diff options
Diffstat (limited to 'drivers/mtd/nand/spi/core.c')
| -rw-r--r-- | drivers/mtd/nand/spi/core.c | 1170 |
1 files changed, 875 insertions, 295 deletions
diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c index 479c2f2cf17f..d207286572d8 100644 --- a/drivers/mtd/nand/spi/core.c +++ b/drivers/mtd/nand/spi/core.c @@ -16,27 +16,13 @@ #include <linux/mtd/spinand.h> #include <linux/of.h> #include <linux/slab.h> +#include <linux/string.h> #include <linux/spi/spi.h> #include <linux/spi/spi-mem.h> -static void spinand_cache_op_adjust_colum(struct spinand_device *spinand, - const struct nand_page_io_req *req, - u16 *column) +int spinand_read_reg_op(struct spinand_device *spinand, u8 reg, u8 *val) { - struct nand_device *nand = spinand_to_nand(spinand); - unsigned int shift; - - if (nand->memorg.planes_per_lun < 2) - return; - - /* The plane number is passed in MSB just above the column address */ - shift = fls(nand->memorg.pagesize); - *column |= req->pos.plane << shift; -} - -static int spinand_read_reg_op(struct spinand_device *spinand, u8 reg, u8 *val) -{ - struct spi_mem_op op = SPINAND_GET_FEATURE_OP(reg, + struct spi_mem_op op = SPINAND_GET_FEATURE_1S_1S_1S_OP(reg, spinand->scratchbuf); int ret; @@ -48,9 +34,9 @@ static int spinand_read_reg_op(struct spinand_device *spinand, u8 reg, u8 *val) return 0; } -static int spinand_write_reg_op(struct spinand_device *spinand, u8 reg, u8 val) +int spinand_write_reg_op(struct spinand_device *spinand, u8 reg, u8 val) { - struct spi_mem_op op = SPINAND_SET_FEATURE_OP(reg, + struct spi_mem_op op = SPINAND_SET_FEATURE_1S_1S_1S_OP(reg, spinand->scratchbuf); *spinand->scratchbuf = val; @@ -152,20 +138,12 @@ int spinand_select_target(struct spinand_device *spinand, unsigned int target) return 0; } -static int spinand_init_cfg_cache(struct spinand_device *spinand) +static int spinand_read_cfg(struct spinand_device *spinand) { struct nand_device *nand = spinand_to_nand(spinand); - struct device *dev = &spinand->spimem->spi->dev; unsigned int target; int ret; - spinand->cfg_cache = devm_kcalloc(dev, - nand->memorg.ntargets, - sizeof(*spinand->cfg_cache), - GFP_KERNEL); - if (!spinand->cfg_cache) - return -ENOMEM; - for (target = 0; target < nand->memorg.ntargets; target++) { ret = spinand_select_target(spinand, target); if (ret) @@ -184,6 +162,21 @@ static int spinand_init_cfg_cache(struct spinand_device *spinand) return 0; } +static int spinand_init_cfg_cache(struct spinand_device *spinand) +{ + struct nand_device *nand = spinand_to_nand(spinand); + struct device *dev = &spinand->spimem->spi->dev; + + spinand->cfg_cache = devm_kcalloc(dev, + nand->memorg.ntargets, + sizeof(*spinand->cfg_cache), + GFP_KERNEL); + if (!spinand->cfg_cache) + return -ENOMEM; + + return 0; +} + static int spinand_init_quad_enable(struct spinand_device *spinand) { bool enable = false; @@ -207,9 +200,169 @@ static int spinand_ecc_enable(struct spinand_device *spinand, enable ? CFG_ECC_ENABLE : 0); } -static int spinand_write_enable_op(struct spinand_device *spinand) +static int spinand_cont_read_enable(struct spinand_device *spinand, + bool enable) +{ + return spinand->set_cont_read(spinand, enable); +} + +static int spinand_check_ecc_status(struct spinand_device *spinand, u8 status) +{ + struct nand_device *nand = spinand_to_nand(spinand); + + if (spinand->eccinfo.get_status) + return spinand->eccinfo.get_status(spinand, status); + + switch (status & STATUS_ECC_MASK) { + case STATUS_ECC_NO_BITFLIPS: + return 0; + + case STATUS_ECC_HAS_BITFLIPS: + /* + * We have no way to know exactly how many bitflips have been + * fixed, so let's return the maximum possible value so that + * wear-leveling layers move the data immediately. + */ + return nanddev_get_ecc_conf(nand)->strength; + + case STATUS_ECC_UNCOR_ERROR: + return -EBADMSG; + + default: + break; + } + + return -EINVAL; +} + +static int spinand_noecc_ooblayout_ecc(struct mtd_info *mtd, int section, + struct mtd_oob_region *region) +{ + return -ERANGE; +} + +static int spinand_noecc_ooblayout_free(struct mtd_info *mtd, int section, + struct mtd_oob_region *region) +{ + if (section) + return -ERANGE; + + /* Reserve 2 bytes for the BBM. */ + region->offset = 2; + region->length = 62; + + return 0; +} + +static const struct mtd_ooblayout_ops spinand_noecc_ooblayout = { + .ecc = spinand_noecc_ooblayout_ecc, + .free = spinand_noecc_ooblayout_free, +}; + +static int spinand_ondie_ecc_init_ctx(struct nand_device *nand) +{ + struct spinand_device *spinand = nand_to_spinand(nand); + struct mtd_info *mtd = nanddev_to_mtd(nand); + struct spinand_ondie_ecc_conf *engine_conf; + + nand->ecc.ctx.conf.engine_type = NAND_ECC_ENGINE_TYPE_ON_DIE; + nand->ecc.ctx.conf.step_size = nand->ecc.requirements.step_size; + nand->ecc.ctx.conf.strength = nand->ecc.requirements.strength; + + engine_conf = kzalloc(sizeof(*engine_conf), GFP_KERNEL); + if (!engine_conf) + return -ENOMEM; + + nand->ecc.ctx.priv = engine_conf; + + if (spinand->eccinfo.ooblayout) + mtd_set_ooblayout(mtd, spinand->eccinfo.ooblayout); + else + mtd_set_ooblayout(mtd, &spinand_noecc_ooblayout); + + return 0; +} + +static void spinand_ondie_ecc_cleanup_ctx(struct nand_device *nand) +{ + kfree(nand->ecc.ctx.priv); +} + +static int spinand_ondie_ecc_prepare_io_req(struct nand_device *nand, + struct nand_page_io_req *req) +{ + struct spinand_device *spinand = nand_to_spinand(nand); + bool enable = (req->mode != MTD_OPS_RAW); + + if (!enable && spinand->flags & SPINAND_NO_RAW_ACCESS) + return -EOPNOTSUPP; + + memset(spinand->oobbuf, 0xff, nanddev_per_page_oobsize(nand)); + + /* Only enable or disable the engine */ + return spinand_ecc_enable(spinand, enable); +} + +static int spinand_ondie_ecc_finish_io_req(struct nand_device *nand, + struct nand_page_io_req *req) +{ + struct spinand_ondie_ecc_conf *engine_conf = nand->ecc.ctx.priv; + struct spinand_device *spinand = nand_to_spinand(nand); + struct mtd_info *mtd = spinand_to_mtd(spinand); + int ret; + + if (req->mode == MTD_OPS_RAW) + return 0; + + /* Nothing to do when finishing a page write */ + if (req->type == NAND_PAGE_WRITE) + return 0; + + /* Finish a page read: check the status, report errors/bitflips */ + ret = spinand_check_ecc_status(spinand, engine_conf->status); + if (ret == -EBADMSG) { + mtd->ecc_stats.failed++; + } else if (ret > 0) { + unsigned int pages; + + /* + * Continuous reads don't allow us to get the detail, + * so we may exagerate the actual number of corrected bitflips. + */ + if (!req->continuous) + pages = 1; + else + pages = req->datalen / nanddev_page_size(nand); + + mtd->ecc_stats.corrected += ret * pages; + } + + return ret; +} + +static const struct nand_ecc_engine_ops spinand_ondie_ecc_engine_ops = { + .init_ctx = spinand_ondie_ecc_init_ctx, + .cleanup_ctx = spinand_ondie_ecc_cleanup_ctx, + .prepare_io_req = spinand_ondie_ecc_prepare_io_req, + .finish_io_req = spinand_ondie_ecc_finish_io_req, +}; + +static struct nand_ecc_engine spinand_ondie_ecc_engine = { + .ops = &spinand_ondie_ecc_engine_ops, +}; + +static void spinand_ondie_ecc_save_status(struct nand_device *nand, u8 status) +{ + struct spinand_ondie_ecc_conf *engine_conf = nand->ecc.ctx.priv; + + if (nand->ecc.ctx.conf.engine_type == NAND_ECC_ENGINE_TYPE_ON_DIE && + engine_conf) + engine_conf->status = status; +} + +int spinand_write_enable_op(struct spinand_device *spinand) { - struct spi_mem_op op = SPINAND_WR_EN_DIS_OP(true); + struct spi_mem_op op = SPINAND_WR_EN_DIS_1S_0_0_OP(true); return spi_mem_exec_op(spinand->spimem, &op); } @@ -219,7 +372,7 @@ static int spinand_load_page_op(struct spinand_device *spinand, { struct nand_device *nand = spinand_to_nand(spinand); unsigned int row = nanddev_pos_to_row(nand, &req->pos); - struct spi_mem_op op = SPINAND_PAGE_READ_OP(row); + struct spi_mem_op op = SPINAND_PAGE_READ_1S_1S_0_OP(row); return spi_mem_exec_op(spinand->spimem, &op); } @@ -227,27 +380,25 @@ static int spinand_load_page_op(struct spinand_device *spinand, static int spinand_read_from_cache_op(struct spinand_device *spinand, const struct nand_page_io_req *req) { - struct spi_mem_op op = *spinand->op_templates.read_cache; struct nand_device *nand = spinand_to_nand(spinand); - struct mtd_info *mtd = nanddev_to_mtd(nand); - struct nand_page_io_req adjreq = *req; + struct mtd_info *mtd = spinand_to_mtd(spinand); + struct spi_mem_dirmap_desc *rdesc; unsigned int nbytes = 0; void *buf = NULL; u16 column = 0; - int ret; + ssize_t ret; if (req->datalen) { - adjreq.datalen = nanddev_page_size(nand); - adjreq.dataoffs = 0; - adjreq.databuf.in = spinand->databuf; buf = spinand->databuf; - nbytes = adjreq.datalen; + if (!req->continuous) + nbytes = nanddev_page_size(nand); + else + nbytes = round_up(req->dataoffs + req->datalen, + nanddev_page_size(nand)); + column = 0; } if (req->ooblen) { - adjreq.ooblen = nanddev_per_page_oobsize(nand); - adjreq.ooboffs = 0; - adjreq.oobbuf.in = spinand->oobbuf; nbytes += nanddev_per_page_oobsize(nand); if (!buf) { buf = spinand->oobbuf; @@ -255,28 +406,40 @@ static int spinand_read_from_cache_op(struct spinand_device *spinand, } } - spinand_cache_op_adjust_colum(spinand, &adjreq, &column); - op.addr.val = column; + if (req->mode == MTD_OPS_RAW) + rdesc = spinand->dirmaps[req->pos.plane].rdesc; + else + rdesc = spinand->dirmaps[req->pos.plane].rdesc_ecc; + + if (spinand->flags & SPINAND_HAS_READ_PLANE_SELECT_BIT) + column |= req->pos.plane << fls(nanddev_page_size(nand)); - /* - * Some controllers are limited in term of max RX data size. In this - * case, just repeat the READ_CACHE operation after updating the - * column. - */ while (nbytes) { - op.data.buf.in = buf; - op.data.nbytes = nbytes; - ret = spi_mem_adjust_op_size(spinand->spimem, &op); - if (ret) + ret = spi_mem_dirmap_read(rdesc, column, nbytes, buf); + if (ret < 0) return ret; - ret = spi_mem_exec_op(spinand->spimem, &op); - if (ret) - return ret; + if (!ret || ret > nbytes) + return -EIO; - buf += op.data.nbytes; - nbytes -= op.data.nbytes; - op.addr.val += op.data.nbytes; + nbytes -= ret; + column += ret; + buf += ret; + + /* + * Dirmap accesses are allowed to toggle the CS. + * Toggling the CS during a continuous read is forbidden. + */ + if (nbytes && req->continuous) { + /* + * Spi controller with broken support of continuous + * reading was detected. Disable future use of + * continuous reading and return -EAGAIN to retry + * reading within regular mode. + */ + spinand->cont_read_possible = false; + return -EAGAIN; + } } if (req->datalen) @@ -300,28 +463,29 @@ static int spinand_read_from_cache_op(struct spinand_device *spinand, static int spinand_write_to_cache_op(struct spinand_device *spinand, const struct nand_page_io_req *req) { - struct spi_mem_op op = *spinand->op_templates.write_cache; struct nand_device *nand = spinand_to_nand(spinand); - struct mtd_info *mtd = nanddev_to_mtd(nand); - struct nand_page_io_req adjreq = *req; - unsigned int nbytes = 0; - void *buf = NULL; - u16 column = 0; - int ret; + struct mtd_info *mtd = spinand_to_mtd(spinand); + struct spi_mem_dirmap_desc *wdesc; + unsigned int nbytes, column = 0; + void *buf = spinand->databuf; + ssize_t ret; - memset(spinand->databuf, 0xff, - nanddev_page_size(nand) + - nanddev_per_page_oobsize(nand)); + /* + * Looks like PROGRAM LOAD (AKA write cache) does not necessarily reset + * the cache content to 0xFF (depends on vendor implementation), so we + * must fill the page cache entirely even if we only want to program + * the data portion of the page, otherwise we might corrupt the BBM or + * user data previously programmed in OOB area. + * + * Only reset the data buffer manually, the OOB buffer is prepared by + * ECC engines ->prepare_io_req() callback. + */ + nbytes = nanddev_page_size(nand) + nanddev_per_page_oobsize(nand); + memset(spinand->databuf, 0xff, nanddev_page_size(nand)); - if (req->datalen) { + if (req->datalen) memcpy(spinand->databuf + req->dataoffs, req->databuf.out, req->datalen); - adjreq.dataoffs = 0; - adjreq.datalen = nanddev_page_size(nand); - adjreq.databuf.out = spinand->databuf; - nbytes = adjreq.datalen; - buf = spinand->databuf; - } if (req->ooblen) { if (req->mode == MTD_OPS_AUTO_OOB) @@ -332,52 +496,27 @@ static int spinand_write_to_cache_op(struct spinand_device *spinand, else memcpy(spinand->oobbuf + req->ooboffs, req->oobbuf.out, req->ooblen); - - adjreq.ooblen = nanddev_per_page_oobsize(nand); - adjreq.ooboffs = 0; - nbytes += nanddev_per_page_oobsize(nand); - if (!buf) { - buf = spinand->oobbuf; - column = nanddev_page_size(nand); - } } - spinand_cache_op_adjust_colum(spinand, &adjreq, &column); + if (req->mode == MTD_OPS_RAW) + wdesc = spinand->dirmaps[req->pos.plane].wdesc; + else + wdesc = spinand->dirmaps[req->pos.plane].wdesc_ecc; - op = *spinand->op_templates.write_cache; - op.addr.val = column; + if (spinand->flags & SPINAND_HAS_PROG_PLANE_SELECT_BIT) + column |= req->pos.plane << fls(nanddev_page_size(nand)); - /* - * Some controllers are limited in term of max TX data size. In this - * case, split the operation into one LOAD CACHE and one or more - * LOAD RANDOM CACHE. - */ while (nbytes) { - op.data.buf.out = buf; - op.data.nbytes = nbytes; - - ret = spi_mem_adjust_op_size(spinand->spimem, &op); - if (ret) + ret = spi_mem_dirmap_write(wdesc, column, nbytes, buf); + if (ret < 0) return ret; - ret = spi_mem_exec_op(spinand->spimem, &op); - if (ret) - return ret; - - buf += op.data.nbytes; - nbytes -= op.data.nbytes; - op.addr.val += op.data.nbytes; + if (!ret || ret > nbytes) + return -EIO; - /* - * We need to use the RANDOM LOAD CACHE operation if there's - * more than one iteration, because the LOAD operation resets - * the cache to 0xff. - */ - if (nbytes) { - column = op.addr.val; - op = *spinand->op_templates.update_cache; - op.addr.val = column; - } + nbytes -= ret; + column += ret; + buf += ret; } return 0; @@ -388,7 +527,7 @@ static int spinand_program_op(struct spinand_device *spinand, { struct nand_device *nand = spinand_to_nand(spinand); unsigned int row = nanddev_pos_to_row(nand, &req->pos); - struct spi_mem_op op = SPINAND_PROG_EXEC_OP(row); + struct spi_mem_op op = SPINAND_PROG_EXEC_1S_1S_0_OP(row); return spi_mem_exec_op(spinand->spimem, &op); } @@ -398,25 +537,41 @@ static int spinand_erase_op(struct spinand_device *spinand, { struct nand_device *nand = spinand_to_nand(spinand); unsigned int row = nanddev_pos_to_row(nand, pos); - struct spi_mem_op op = SPINAND_BLK_ERASE_OP(row); + struct spi_mem_op op = SPINAND_BLK_ERASE_1S_1S_0_OP(row); return spi_mem_exec_op(spinand->spimem, &op); } -static int spinand_wait(struct spinand_device *spinand, u8 *s) +/** + * spinand_wait() - Poll memory device status + * @spinand: the spinand device + * @initial_delay_us: delay in us before starting to poll + * @poll_delay_us: time to sleep between reads in us + * @s: the pointer to variable to store the value of REG_STATUS + * + * This function polls a status register (REG_STATUS) and returns when + * the STATUS_READY bit is 0 or when the timeout has expired. + * + * Return: 0 on success, a negative error code otherwise. + */ +int spinand_wait(struct spinand_device *spinand, unsigned long initial_delay_us, + unsigned long poll_delay_us, u8 *s) { - unsigned long timeo = jiffies + msecs_to_jiffies(400); + struct spi_mem_op op = SPINAND_GET_FEATURE_1S_1S_1S_OP(REG_STATUS, + spinand->scratchbuf); u8 status; int ret; - do { - ret = spinand_read_status(spinand, &status); - if (ret) - return ret; + ret = spi_mem_poll_status(spinand->spimem, &op, STATUS_BUSY, 0, + initial_delay_us, + poll_delay_us, + SPINAND_WAITRDY_TIMEOUT_MS); + if (ret) + return ret; - if (!(status & STATUS_BUSY)) - goto out; - } while (time_before(jiffies, timeo)); + status = *spinand->scratchbuf; + if (!(status & STATUS_BUSY)) + goto out; /* * Extra read, just in case the STATUS_READY bit has changed @@ -433,10 +588,11 @@ out: return status & STATUS_BUSY ? -ETIMEDOUT : 0; } -static int spinand_read_id_op(struct spinand_device *spinand, u8 *buf) +static int spinand_read_id_op(struct spinand_device *spinand, u8 naddr, + u8 ndummy, u8 *buf) { - struct spi_mem_op op = SPINAND_READID_OP(0, spinand->scratchbuf, - SPINAND_MAX_ID_LEN); + struct spi_mem_op op = SPINAND_READID_1S_1S_1S_OP( + naddr, ndummy, spinand->scratchbuf, SPINAND_MAX_ID_LEN); int ret; ret = spi_mem_exec_op(spinand->spimem, &op); @@ -448,14 +604,17 @@ static int spinand_read_id_op(struct spinand_device *spinand, u8 *buf) static int spinand_reset_op(struct spinand_device *spinand) { - struct spi_mem_op op = SPINAND_RESET_OP; + struct spi_mem_op op = SPINAND_RESET_1S_0_0_OP; int ret; ret = spi_mem_exec_op(spinand->spimem, &op); if (ret) return ret; - return spinand_wait(spinand, NULL); + return spinand_wait(spinand, + SPINAND_RESET_INITIAL_DELAY_US, + SPINAND_RESET_POLL_DELAY_US, + NULL); } static int spinand_lock_block(struct spinand_device *spinand, u8 lock) @@ -463,66 +622,64 @@ static int spinand_lock_block(struct spinand_device *spinand, u8 lock) return spinand_write_reg_op(spinand, REG_BLOCK_LOCK, lock); } -static int spinand_check_ecc_status(struct spinand_device *spinand, u8 status) +/** + * spinand_read_page() - Read a page + * @spinand: the spinand device + * @req: the I/O request + * + * Return: 0 or a positive number of bitflips corrected on success. + * A negative error code otherwise. + */ +int spinand_read_page(struct spinand_device *spinand, + const struct nand_page_io_req *req) { struct nand_device *nand = spinand_to_nand(spinand); - - if (spinand->eccinfo.get_status) - return spinand->eccinfo.get_status(spinand, status); - - switch (status & STATUS_ECC_MASK) { - case STATUS_ECC_NO_BITFLIPS: - return 0; - - case STATUS_ECC_HAS_BITFLIPS: - /* - * We have no way to know exactly how many bitflips have been - * fixed, so let's return the maximum possible value so that - * wear-leveling layers move the data immediately. - */ - return nand->eccreq.strength; - - case STATUS_ECC_UNCOR_ERROR: - return -EBADMSG; - - default: - break; - } - - return -EINVAL; -} - -static int spinand_read_page(struct spinand_device *spinand, - const struct nand_page_io_req *req, - bool ecc_enabled) -{ u8 status; int ret; + ret = nand_ecc_prepare_io_req(nand, (struct nand_page_io_req *)req); + if (ret) + return ret; + ret = spinand_load_page_op(spinand, req); if (ret) return ret; - ret = spinand_wait(spinand, &status); + ret = spinand_wait(spinand, + SPINAND_READ_INITIAL_DELAY_US, + SPINAND_READ_POLL_DELAY_US, + &status); if (ret < 0) return ret; + spinand_ondie_ecc_save_status(nand, status); + ret = spinand_read_from_cache_op(spinand, req); if (ret) return ret; - if (!ecc_enabled) - return 0; - - return spinand_check_ecc_status(spinand, status); + return nand_ecc_finish_io_req(nand, (struct nand_page_io_req *)req); } -static int spinand_write_page(struct spinand_device *spinand, - const struct nand_page_io_req *req) +/** + * spinand_write_page() - Write a page + * @spinand: the spinand device + * @req: the I/O request + * + * Return: 0 or a positive number of bitflips corrected on success. + * A negative error code otherwise. + */ +int spinand_write_page(struct spinand_device *spinand, + const struct nand_page_io_req *req) { + struct nand_device *nand = spinand_to_nand(spinand); u8 status; int ret; + ret = nand_ecc_prepare_io_req(nand, (struct nand_page_io_req *)req); + if (ret) + return ret; + ret = spinand_write_enable_op(spinand); if (ret) return ret; @@ -535,60 +692,244 @@ static int spinand_write_page(struct spinand_device *spinand, if (ret) return ret; - ret = spinand_wait(spinand, &status); - if (!ret && (status & STATUS_PROG_FAILED)) - ret = -EIO; + ret = spinand_wait(spinand, + SPINAND_WRITE_INITIAL_DELAY_US, + SPINAND_WRITE_POLL_DELAY_US, + &status); + if (ret) + return ret; - return ret; + if (status & STATUS_PROG_FAILED) + return -EIO; + + return nand_ecc_finish_io_req(nand, (struct nand_page_io_req *)req); } -static int spinand_mtd_read(struct mtd_info *mtd, loff_t from, - struct mtd_oob_ops *ops) +static int spinand_mtd_regular_page_read(struct mtd_info *mtd, loff_t from, + struct mtd_oob_ops *ops, + unsigned int *max_bitflips) { struct spinand_device *spinand = mtd_to_spinand(mtd); struct nand_device *nand = mtd_to_nanddev(mtd); - unsigned int max_bitflips = 0; + struct mtd_ecc_stats old_stats; struct nand_io_iter iter; - bool enable_ecc = false; + bool disable_ecc = false; bool ecc_failed = false; - int ret = 0; + unsigned int retry_mode = 0; + int ret; - if (ops->mode != MTD_OPS_RAW && spinand->eccinfo.ooblayout) - enable_ecc = true; + old_stats = mtd->ecc_stats; - mutex_lock(&spinand->lock); + if (ops->mode == MTD_OPS_RAW || !mtd->ooblayout) + disable_ecc = true; - nanddev_io_for_each_page(nand, from, ops, &iter) { - ret = spinand_select_target(spinand, iter.req.pos.target); - if (ret) - break; + nanddev_io_for_each_page(nand, NAND_PAGE_READ, from, ops, &iter) { + if (disable_ecc) + iter.req.mode = MTD_OPS_RAW; - ret = spinand_ecc_enable(spinand, enable_ecc); + ret = spinand_select_target(spinand, iter.req.pos.target); if (ret) break; - ret = spinand_read_page(spinand, &iter.req, enable_ecc); +read_retry: + ret = spinand_read_page(spinand, &iter.req); if (ret < 0 && ret != -EBADMSG) break; - if (ret == -EBADMSG) { + if (ret == -EBADMSG && spinand->set_read_retry) { + if (spinand->read_retries && (++retry_mode <= spinand->read_retries)) { + ret = spinand->set_read_retry(spinand, retry_mode); + if (ret < 0) { + spinand->set_read_retry(spinand, 0); + return ret; + } + + /* Reset ecc_stats; retry */ + mtd->ecc_stats = old_stats; + goto read_retry; + } else { + /* No more retry modes; real failure */ + ecc_failed = true; + } + } else if (ret == -EBADMSG) { ecc_failed = true; - mtd->ecc_stats.failed++; - ret = 0; } else { - mtd->ecc_stats.corrected += ret; - max_bitflips = max_t(unsigned int, max_bitflips, ret); + *max_bitflips = max_t(unsigned int, *max_bitflips, ret); } + ret = 0; ops->retlen += iter.req.datalen; ops->oobretlen += iter.req.ooblen; - } - mutex_unlock(&spinand->lock); + /* Reset to retry mode 0 */ + if (retry_mode) { + retry_mode = 0; + ret = spinand->set_read_retry(spinand, retry_mode); + if (ret < 0) + return ret; + } + } if (ecc_failed && !ret) ret = -EBADMSG; + return ret; +} + +static int spinand_mtd_continuous_page_read(struct mtd_info *mtd, loff_t from, + struct mtd_oob_ops *ops, + unsigned int *max_bitflips) +{ + struct spinand_device *spinand = mtd_to_spinand(mtd); + struct nand_device *nand = mtd_to_nanddev(mtd); + struct nand_io_iter iter; + u8 status; + int ret; + + ret = spinand_cont_read_enable(spinand, true); + if (ret) + return ret; + + /* + * The cache is divided into two halves. While one half of the cache has + * the requested data, the other half is loaded with the next chunk of data. + * Therefore, the host can read out the data continuously from page to page. + * Each data read must be a multiple of 4-bytes and full pages should be read; + * otherwise, the data output might get out of sequence from one read command + * to another. + */ + nanddev_io_for_each_block(nand, NAND_PAGE_READ, from, ops, &iter) { + ret = spinand_select_target(spinand, iter.req.pos.target); + if (ret) + goto end_cont_read; + + ret = nand_ecc_prepare_io_req(nand, &iter.req); + if (ret) + goto end_cont_read; + + ret = spinand_load_page_op(spinand, &iter.req); + if (ret) + goto end_cont_read; + + ret = spinand_wait(spinand, SPINAND_READ_INITIAL_DELAY_US, + SPINAND_READ_POLL_DELAY_US, NULL); + if (ret < 0) + goto end_cont_read; + + ret = spinand_read_from_cache_op(spinand, &iter.req); + if (ret) + goto end_cont_read; + + ops->retlen += iter.req.datalen; + + ret = spinand_read_status(spinand, &status); + if (ret) + goto end_cont_read; + + spinand_ondie_ecc_save_status(nand, status); + + ret = nand_ecc_finish_io_req(nand, &iter.req); + if (ret < 0) + goto end_cont_read; + + *max_bitflips = max_t(unsigned int, *max_bitflips, ret); + ret = 0; + } + +end_cont_read: + /* + * Once all the data has been read out, the host can either pull CS# + * high and wait for tRST or manually clear the bit in the configuration + * register to terminate the continuous read operation. We have no + * guarantee the SPI controller drivers will effectively deassert the CS + * when we expect them to, so take the register based approach. + */ + spinand_cont_read_enable(spinand, false); + + return ret; +} + +static void spinand_cont_read_init(struct spinand_device *spinand) +{ + struct nand_device *nand = spinand_to_nand(spinand); + enum nand_ecc_engine_type engine_type = nand->ecc.ctx.conf.engine_type; + + /* OOBs cannot be retrieved so external/on-host ECC engine won't work */ + if (spinand->set_cont_read && + (engine_type == NAND_ECC_ENGINE_TYPE_ON_DIE || + engine_type == NAND_ECC_ENGINE_TYPE_NONE)) { + spinand->cont_read_possible = true; + } +} + +static bool spinand_use_cont_read(struct mtd_info *mtd, loff_t from, + struct mtd_oob_ops *ops) +{ + struct nand_device *nand = mtd_to_nanddev(mtd); + struct spinand_device *spinand = nand_to_spinand(nand); + struct nand_pos start_pos, end_pos; + + if (!spinand->cont_read_possible) + return false; + + /* OOBs won't be retrieved */ + if (ops->ooblen || ops->oobbuf) + return false; + + nanddev_offs_to_pos(nand, from, &start_pos); + nanddev_offs_to_pos(nand, from + ops->len - 1, &end_pos); + + /* + * Continuous reads never cross LUN boundaries. Some devices don't + * support crossing planes boundaries. Some devices don't even support + * crossing blocks boundaries. The common case being to read through UBI, + * we will very rarely read two consequent blocks or more, so it is safer + * and easier (can be improved) to only enable continuous reads when + * reading within the same erase block. + */ + if (start_pos.target != end_pos.target || + start_pos.plane != end_pos.plane || + start_pos.eraseblock != end_pos.eraseblock) + return false; + + return start_pos.page < end_pos.page; +} + +static int spinand_mtd_read(struct mtd_info *mtd, loff_t from, + struct mtd_oob_ops *ops) +{ + struct spinand_device *spinand = mtd_to_spinand(mtd); + struct mtd_ecc_stats old_stats; + unsigned int max_bitflips = 0; + int ret; + + mutex_lock(&spinand->lock); + + old_stats = mtd->ecc_stats; + + if (spinand_use_cont_read(mtd, from, ops)) { + ret = spinand_mtd_continuous_page_read(mtd, from, ops, &max_bitflips); + if (ret == -EAGAIN && !spinand->cont_read_possible) { + /* + * Spi controller with broken support of continuous + * reading was detected (see spinand_read_from_cache_op()), + * repeat reading in regular mode. + */ + ret = spinand_mtd_regular_page_read(mtd, from, ops, &max_bitflips); + } + } else { + ret = spinand_mtd_regular_page_read(mtd, from, ops, &max_bitflips); + } + + if (ops->stats) { + ops->stats->uncorrectable_errors += + mtd->ecc_stats.failed - old_stats.failed; + ops->stats->corrected_bitflips += + mtd->ecc_stats.corrected - old_stats.corrected; + } + + mutex_unlock(&spinand->lock); + return ret ? ret : max_bitflips; } @@ -598,20 +939,19 @@ static int spinand_mtd_write(struct mtd_info *mtd, loff_t to, struct spinand_device *spinand = mtd_to_spinand(mtd); struct nand_device *nand = mtd_to_nanddev(mtd); struct nand_io_iter iter; - bool enable_ecc = false; + bool disable_ecc = false; int ret = 0; - if (ops->mode != MTD_OPS_RAW && mtd->ooblayout) - enable_ecc = true; + if (ops->mode == MTD_OPS_RAW || !mtd->ooblayout) + disable_ecc = true; mutex_lock(&spinand->lock); - nanddev_io_for_each_page(nand, to, ops, &iter) { - ret = spinand_select_target(spinand, iter.req.pos.target); - if (ret) - break; + nanddev_io_for_each_page(nand, NAND_PAGE_WRITE, to, ops, &iter) { + if (disable_ecc) + iter.req.mode = MTD_OPS_RAW; - ret = spinand_ecc_enable(spinand, enable_ecc); + ret = spinand_select_target(spinand, iter.req.pos.target); if (ret) break; @@ -631,18 +971,26 @@ static int spinand_mtd_write(struct mtd_info *mtd, loff_t to, static bool spinand_isbad(struct nand_device *nand, const struct nand_pos *pos) { struct spinand_device *spinand = nand_to_spinand(nand); + u8 marker[2] = { }; struct nand_page_io_req req = { .pos = *pos, - .ooblen = 2, + .ooblen = sizeof(marker), .ooboffs = 0, - .oobbuf.in = spinand->oobbuf, + .oobbuf.in = marker, .mode = MTD_OPS_RAW, }; + int ret; - memset(spinand->oobbuf, 0, 2); spinand_select_target(spinand, pos->target); - spinand_read_page(spinand, &req, false); - if (spinand->oobbuf[0] != 0xff || spinand->oobbuf[1] != 0xff) + + ret = spinand_read_page(spinand, &req); + if (ret == -EOPNOTSUPP) { + /* Retry with ECC in case raw access is not supported */ + req.mode = MTD_OPS_PLACE_OOB; + spinand_read_page(spinand, &req); + } + + if (marker[0] != 0xff || marker[1] != 0xff) return true; return false; @@ -666,27 +1014,28 @@ static int spinand_mtd_block_isbad(struct mtd_info *mtd, loff_t offs) static int spinand_markbad(struct nand_device *nand, const struct nand_pos *pos) { struct spinand_device *spinand = nand_to_spinand(nand); + u8 marker[2] = { }; struct nand_page_io_req req = { .pos = *pos, .ooboffs = 0, - .ooblen = 2, - .oobbuf.out = spinand->oobbuf, + .ooblen = sizeof(marker), + .oobbuf.out = marker, + .mode = MTD_OPS_RAW, }; int ret; - /* Erase block before marking it bad. */ ret = spinand_select_target(spinand, pos->target); if (ret) return ret; - ret = spinand_write_enable_op(spinand); - if (ret) - return ret; - - spinand_erase_op(spinand, pos); + ret = spinand_write_page(spinand, &req); + if (ret == -EOPNOTSUPP) { + /* Retry with ECC in case raw access is not supported */ + req.mode = MTD_OPS_PLACE_OOB; + ret = spinand_write_page(spinand, &req); + } - memset(spinand->oobbuf, 0, 2); - return spinand_write_page(spinand, &req); + return ret; } static int spinand_mtd_block_markbad(struct mtd_info *mtd, loff_t offs) @@ -722,7 +1071,11 @@ static int spinand_erase(struct nand_device *nand, const struct nand_pos *pos) if (ret) return ret; - ret = spinand_wait(spinand, &status); + ret = spinand_wait(spinand, + SPINAND_ERASE_INITIAL_DELAY_US, + SPINAND_ERASE_POLL_DELAY_US, + &status); + if (!ret && (status & STATUS_ERASE_FAILED)) ret = -EIO; @@ -757,6 +1110,114 @@ static int spinand_mtd_block_isreserved(struct mtd_info *mtd, loff_t offs) return ret; } +static struct spi_mem_dirmap_desc *spinand_create_rdesc( + struct spinand_device *spinand, + struct spi_mem_dirmap_info *info) +{ + struct nand_device *nand = spinand_to_nand(spinand); + struct spi_mem_dirmap_desc *desc = NULL; + + if (spinand->cont_read_possible) { + /* + * spi controller may return an error if info->length is + * too large + */ + info->length = nanddev_eraseblock_size(nand); + desc = devm_spi_mem_dirmap_create(&spinand->spimem->spi->dev, + spinand->spimem, info); + } + + if (IS_ERR_OR_NULL(desc)) { + /* + * continuous reading is not supported by flash or + * its spi controller, use regular reading + */ + spinand->cont_read_possible = false; + + info->length = nanddev_page_size(nand) + + nanddev_per_page_oobsize(nand); + desc = devm_spi_mem_dirmap_create(&spinand->spimem->spi->dev, + spinand->spimem, info); + } + + return desc; +} + +static int spinand_create_dirmap(struct spinand_device *spinand, + unsigned int plane) +{ + struct nand_device *nand = spinand_to_nand(spinand); + struct spi_mem_dirmap_info info = { 0 }; + struct spi_mem_dirmap_desc *desc; + + /* The plane number is passed in MSB just above the column address */ + info.offset = plane << fls(nand->memorg.pagesize); + + info.length = nanddev_page_size(nand) + nanddev_per_page_oobsize(nand); + info.op_tmpl = *spinand->op_templates.update_cache; + desc = devm_spi_mem_dirmap_create(&spinand->spimem->spi->dev, + spinand->spimem, &info); + if (IS_ERR(desc)) + return PTR_ERR(desc); + + spinand->dirmaps[plane].wdesc = desc; + + info.op_tmpl = *spinand->op_templates.read_cache; + desc = spinand_create_rdesc(spinand, &info); + if (IS_ERR(desc)) + return PTR_ERR(desc); + + spinand->dirmaps[plane].rdesc = desc; + + if (nand->ecc.engine->integration != NAND_ECC_ENGINE_INTEGRATION_PIPELINED) { + spinand->dirmaps[plane].wdesc_ecc = spinand->dirmaps[plane].wdesc; + spinand->dirmaps[plane].rdesc_ecc = spinand->dirmaps[plane].rdesc; + + return 0; + } + + info.length = nanddev_page_size(nand) + nanddev_per_page_oobsize(nand); + info.op_tmpl = *spinand->op_templates.update_cache; + info.op_tmpl.data.ecc = true; + desc = devm_spi_mem_dirmap_create(&spinand->spimem->spi->dev, + spinand->spimem, &info); + if (IS_ERR(desc)) + return PTR_ERR(desc); + + spinand->dirmaps[plane].wdesc_ecc = desc; + + info.op_tmpl = *spinand->op_templates.read_cache; + info.op_tmpl.data.ecc = true; + desc = spinand_create_rdesc(spinand, &info); + if (IS_ERR(desc)) + return PTR_ERR(desc); + + spinand->dirmaps[plane].rdesc_ecc = desc; + + return 0; +} + +static int spinand_create_dirmaps(struct spinand_device *spinand) +{ + struct nand_device *nand = spinand_to_nand(spinand); + int i, ret; + + spinand->dirmaps = devm_kzalloc(&spinand->spimem->spi->dev, + sizeof(*spinand->dirmaps) * + nand->memorg.planes_per_lun, + GFP_KERNEL); + if (!spinand->dirmaps) + return -ENOMEM; + + for (i = 0; i < nand->memorg.planes_per_lun; i++) { + ret = spinand_create_dirmap(spinand, i); + if (ret) + return ret; + } + + return 0; +} + static const struct nand_ops spinand_ops = { .erase = spinand_erase, .markbad = spinand_markbad, @@ -764,35 +1225,93 @@ static const struct nand_ops spinand_ops = { }; static const struct spinand_manufacturer *spinand_manufacturers[] = { + &alliancememory_spinand_manufacturer, + &ato_spinand_manufacturer, + &esmt_8c_spinand_manufacturer, + &esmt_c8_spinand_manufacturer, + &fmsh_spinand_manufacturer, + &foresee_spinand_manufacturer, &gigadevice_spinand_manufacturer, ¯onix_spinand_manufacturer, µn_spinand_manufacturer, + ¶gon_spinand_manufacturer, + &skyhigh_spinand_manufacturer, &toshiba_spinand_manufacturer, &winbond_spinand_manufacturer, + &xtx_spinand_manufacturer, }; -static int spinand_manufacturer_detect(struct spinand_device *spinand) +static int spinand_manufacturer_match(struct spinand_device *spinand, + enum spinand_readid_method rdid_method) { + u8 *id = spinand->id.data; unsigned int i; int ret; for (i = 0; i < ARRAY_SIZE(spinand_manufacturers); i++) { - ret = spinand_manufacturers[i]->ops->detect(spinand); - if (ret > 0) { - spinand->manufacturer = spinand_manufacturers[i]; - return 0; - } else if (ret < 0) { - return ret; - } + const struct spinand_manufacturer *manufacturer = + spinand_manufacturers[i]; + + if (id[0] != manufacturer->id) + continue; + + ret = spinand_match_and_init(spinand, + manufacturer->chips, + manufacturer->nchips, + rdid_method); + if (ret < 0) + continue; + + spinand->manufacturer = manufacturer; + return 0; } + return -EOPNOTSUPP; +} - return -ENOTSUPP; +static int spinand_id_detect(struct spinand_device *spinand) +{ + u8 *id = spinand->id.data; + int ret; + + ret = spinand_read_id_op(spinand, 0, 0, id); + if (ret) + return ret; + ret = spinand_manufacturer_match(spinand, SPINAND_READID_METHOD_OPCODE); + if (!ret) + return 0; + + ret = spinand_read_id_op(spinand, 1, 0, id); + if (ret) + return ret; + ret = spinand_manufacturer_match(spinand, + SPINAND_READID_METHOD_OPCODE_ADDR); + if (!ret) + return 0; + + ret = spinand_read_id_op(spinand, 0, 1, id); + if (ret) + return ret; + ret = spinand_manufacturer_match(spinand, + SPINAND_READID_METHOD_OPCODE_DUMMY); + + return ret; } static int spinand_manufacturer_init(struct spinand_device *spinand) { - if (spinand->manufacturer->ops->init) - return spinand->manufacturer->ops->init(spinand); + int ret; + + if (spinand->manufacturer->ops->init) { + ret = spinand->manufacturer->ops->init(spinand); + if (ret) + return ret; + } + + if (spinand->configure_chip) { + ret = spinand->configure_chip(spinand); + if (ret) + return ret; + } return 0; } @@ -809,10 +1328,13 @@ spinand_select_op_variant(struct spinand_device *spinand, const struct spinand_op_variants *variants) { struct nand_device *nand = spinand_to_nand(spinand); + const struct spi_mem_op *best_variant = NULL; + u64 best_op_duration_ns = ULLONG_MAX; unsigned int i; for (i = 0; i < variants->nops; i++) { struct spi_mem_op op = variants->ops[i]; + u64 op_duration_ns = 0; unsigned int nbytes; int ret; @@ -825,17 +1347,23 @@ spinand_select_op_variant(struct spinand_device *spinand, if (ret) break; + spi_mem_adjust_op_freq(spinand->spimem, &op); + if (!spi_mem_supports_op(spinand->spimem, &op)) break; nbytes -= op.data.nbytes; + + op_duration_ns += spi_mem_calc_op_duration(spinand->spimem, &op); } - if (!nbytes) - return &variants->ops[i]; + if (!nbytes && op_duration_ns < best_op_duration_ns) { + best_op_duration_ns = op_duration_ns; + best_variant = &variants->ops[i]; + } } - return NULL; + return best_variant; } /** @@ -844,9 +1372,9 @@ spinand_select_op_variant(struct spinand_device *spinand, * @spinand: SPI NAND object * @table: SPI NAND device description table * @table_size: size of the device description table + * @rdid_method: read id method to match * - * Should be used by SPI NAND manufacturer drivers when they want to find a - * match between a device ID retrieved through the READ_ID command and an + * Match between a device ID retrieved through the READ_ID command and an * entry in the SPI NAND description table. If a match is found, the spinand * object will be initialized with information provided by the matching * spinand_info entry. @@ -855,8 +1383,10 @@ spinand_select_op_variant(struct spinand_device *spinand, */ int spinand_match_and_init(struct spinand_device *spinand, const struct spinand_info *table, - unsigned int table_size, u8 devid) + unsigned int table_size, + enum spinand_readid_method rdid_method) { + u8 *id = spinand->id.data; struct nand_device *nand = spinand_to_nand(spinand); unsigned int i; @@ -864,14 +1394,24 @@ int spinand_match_and_init(struct spinand_device *spinand, const struct spinand_info *info = &table[i]; const struct spi_mem_op *op; - if (devid != info->devid) + if (rdid_method != info->devid.method) + continue; + + if (memcmp(id + 1, info->devid.id, info->devid.len)) continue; nand->memorg = table[i].memorg; - nand->eccreq = table[i].eccreq; + nanddev_set_ecc_requirements(nand, &table[i].eccreq); spinand->eccinfo = table[i].eccinfo; spinand->flags = table[i].flags; + spinand->id.len = 1 + table[i].devid.len; spinand->select_target = table[i].select_target; + spinand->configure_chip = table[i].configure_chip; + spinand->set_cont_read = table[i].set_cont_read; + spinand->fact_otp = &table[i].fact_otp; + spinand->user_otp = &table[i].user_otp; + spinand->read_retries = table[i].read_retries; + spinand->set_read_retry = table[i].set_read_retry; op = spinand_select_op_variant(spinand, info->op_variants.read_cache); @@ -907,13 +1447,7 @@ static int spinand_detect(struct spinand_device *spinand) if (ret) return ret; - ret = spinand_read_id_op(spinand, spinand->id.data); - if (ret) - return ret; - - spinand->id.len = SPINAND_MAX_ID_LEN; - - ret = spinand_manufacturer_detect(spinand); + ret = spinand_id_detect(spinand); if (ret) { dev_err(dev, "unknown raw ID %*phN\n", SPINAND_MAX_ID_LEN, spinand->id.data); @@ -936,36 +1470,71 @@ static int spinand_detect(struct spinand_device *spinand) return 0; } -static int spinand_noecc_ooblayout_ecc(struct mtd_info *mtd, int section, - struct mtd_oob_region *region) +static int spinand_init_flash(struct spinand_device *spinand) { - return -ERANGE; + struct device *dev = &spinand->spimem->spi->dev; + struct nand_device *nand = spinand_to_nand(spinand); + int ret, i; + + ret = spinand_read_cfg(spinand); + if (ret) + return ret; + + ret = spinand_init_quad_enable(spinand); + if (ret) + return ret; + + ret = spinand_upd_cfg(spinand, CFG_OTP_ENABLE, 0); + if (ret) + return ret; + + ret = spinand_manufacturer_init(spinand); + if (ret) { + dev_err(dev, + "Failed to initialize the SPI NAND chip (err = %d)\n", + ret); + return ret; + } + + /* After power up, all blocks are locked, so unlock them here. */ + for (i = 0; i < nand->memorg.ntargets; i++) { + ret = spinand_select_target(spinand, i); + if (ret) + break; + + ret = spinand_lock_block(spinand, BL_ALL_UNLOCKED); + if (ret) + break; + } + + if (ret) + spinand_manufacturer_cleanup(spinand); + + return ret; } -static int spinand_noecc_ooblayout_free(struct mtd_info *mtd, int section, - struct mtd_oob_region *region) +static void spinand_mtd_resume(struct mtd_info *mtd) { - if (section) - return -ERANGE; + struct spinand_device *spinand = mtd_to_spinand(mtd); + int ret; - /* Reserve 2 bytes for the BBM. */ - region->offset = 2; - region->length = 62; + ret = spinand_reset_op(spinand); + if (ret) + return; - return 0; -} + ret = spinand_init_flash(spinand); + if (ret) + return; -static const struct mtd_ooblayout_ops spinand_noecc_ooblayout = { - .ecc = spinand_noecc_ooblayout_ecc, - .free = spinand_noecc_ooblayout_free, -}; + spinand_ecc_enable(spinand, false); +} static int spinand_init(struct spinand_device *spinand) { struct device *dev = &spinand->spimem->spi->dev; struct mtd_info *mtd = spinand_to_mtd(spinand); struct nand_device *nand = mtd_to_nanddev(mtd); - int ret, i; + int ret; /* * We need a scratch buffer because the spi_mem interface requires that @@ -984,9 +1553,8 @@ static int spinand_init(struct spinand_device *spinand) * may use this buffer for DMA access. * Memory allocated by devm_ does not guarantee DMA-safe alignment. */ - spinand->databuf = kzalloc(nanddev_page_size(nand) + - nanddev_per_page_oobsize(nand), - GFP_KERNEL); + spinand->databuf = kzalloc(nanddev_eraseblock_size(nand), + GFP_KERNEL); if (!spinand->databuf) { ret = -ENOMEM; goto err_free_bufs; @@ -998,61 +1566,70 @@ static int spinand_init(struct spinand_device *spinand) if (ret) goto err_free_bufs; - ret = spinand_init_quad_enable(spinand); + ret = spinand_init_flash(spinand); if (ret) goto err_free_bufs; - ret = spinand_upd_cfg(spinand, CFG_OTP_ENABLE, 0); + ret = nanddev_init(nand, &spinand_ops, THIS_MODULE); if (ret) - goto err_free_bufs; - - ret = spinand_manufacturer_init(spinand); - if (ret) { - dev_err(dev, - "Failed to initialize the SPI NAND chip (err = %d)\n", - ret); - goto err_free_bufs; - } - - /* After power up, all blocks are locked, so unlock them here. */ - for (i = 0; i < nand->memorg.ntargets; i++) { - ret = spinand_select_target(spinand, i); - if (ret) - goto err_free_bufs; + goto err_manuf_cleanup; - ret = spinand_lock_block(spinand, BL_ALL_UNLOCKED); - if (ret) - goto err_free_bufs; - } + /* SPI-NAND default ECC engine is on-die */ + nand->ecc.defaults.engine_type = NAND_ECC_ENGINE_TYPE_ON_DIE; + nand->ecc.ondie_engine = &spinand_ondie_ecc_engine; - ret = nanddev_init(nand, &spinand_ops, THIS_MODULE); + spinand_ecc_enable(spinand, false); + ret = nanddev_ecc_engine_init(nand); if (ret) - goto err_manuf_cleanup; + goto err_cleanup_nanddev; /* - * Right now, we don't support ECC, so let the whole oob - * area is available for user. + * Continuous read can only be enabled with an on-die ECC engine, so the + * ECC initialization must have happened previously. */ + spinand_cont_read_init(spinand); + mtd->_read_oob = spinand_mtd_read; mtd->_write_oob = spinand_mtd_write; mtd->_block_isbad = spinand_mtd_block_isbad; mtd->_block_markbad = spinand_mtd_block_markbad; mtd->_block_isreserved = spinand_mtd_block_isreserved; mtd->_erase = spinand_mtd_erase; + mtd->_max_bad_blocks = nanddev_mtd_max_bad_blocks; + mtd->_resume = spinand_mtd_resume; - if (spinand->eccinfo.ooblayout) - mtd_set_ooblayout(mtd, spinand->eccinfo.ooblayout); - else - mtd_set_ooblayout(mtd, &spinand_noecc_ooblayout); + if (spinand_user_otp_size(spinand) || spinand_fact_otp_size(spinand)) { + ret = spinand_set_mtd_otp_ops(spinand); + if (ret) + goto err_cleanup_ecc_engine; + } - ret = mtd_ooblayout_count_freebytes(mtd); - if (ret < 0) - goto err_cleanup_nanddev; + if (nand->ecc.engine) { + ret = mtd_ooblayout_count_freebytes(mtd); + if (ret < 0) + goto err_cleanup_ecc_engine; + } mtd->oobavail = ret; + /* Propagate ECC information to mtd_info */ + mtd->ecc_strength = nanddev_get_ecc_conf(nand)->strength; + mtd->ecc_step_size = nanddev_get_ecc_conf(nand)->step_size; + mtd->bitflip_threshold = DIV_ROUND_UP(mtd->ecc_strength * 3, 4); + + ret = spinand_create_dirmaps(spinand); + if (ret) { + dev_err(dev, + "Failed to create direct mappings for read/write operations (err = %d)\n", + ret); + goto err_cleanup_ecc_engine; + } + return 0; +err_cleanup_ecc_engine: + nanddev_ecc_engine_cleanup(nand); + err_cleanup_nanddev: nanddev_cleanup(nand); @@ -1069,6 +1646,7 @@ static void spinand_cleanup(struct spinand_device *spinand) { struct nand_device *nand = spinand_to_nand(spinand); + nanddev_ecc_engine_cleanup(nand); nanddev_cleanup(nand); spinand_manufacturer_cleanup(spinand); kfree(spinand->databuf); @@ -1131,12 +1709,14 @@ static const struct spi_device_id spinand_ids[] = { { .name = "spi-nand" }, { /* sentinel */ }, }; +MODULE_DEVICE_TABLE(spi, spinand_ids); #ifdef CONFIG_OF static const struct of_device_id spinand_of_ids[] = { { .compatible = "spi-nand" }, { /* sentinel */ }, }; +MODULE_DEVICE_TABLE(of, spinand_of_ids); #endif static struct spi_mem_driver spinand_drv = { |