diff options
Diffstat (limited to 'fs/xfs/xfs_iomap.c')
-rw-r--r-- | fs/xfs/xfs_iomap.c | 372 |
1 files changed, 224 insertions, 148 deletions
diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index 18c8f168b153..d61460309a78 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -24,9 +24,12 @@ #include "xfs_iomap.h" #include "xfs_trace.h" #include "xfs_quota.h" +#include "xfs_rtgroup.h" #include "xfs_dquot_item.h" #include "xfs_dquot.h" #include "xfs_reflink.h" +#include "xfs_health.h" +#include "xfs_rtbitmap.h" #define XFS_ALLOC_ALIGN(mp, off) \ (((off) >> mp->m_allocsize_log) << mp->m_allocsize_log) @@ -45,6 +48,7 @@ xfs_alert_fsblock_zero( (unsigned long long)imap->br_startoff, (unsigned long long)imap->br_blockcount, imap->br_state); + xfs_bmap_mark_sick(ip, XFS_DATA_FORK); return -EFSCORRUPTED; } @@ -99,8 +103,10 @@ xfs_bmbt_to_iomap( struct xfs_mount *mp = ip->i_mount; struct xfs_buftarg *target = xfs_inode_buftarg(ip); - if (unlikely(!xfs_valid_startblock(ip, imap->br_startblock))) + if (unlikely(!xfs_valid_startblock(ip, imap->br_startblock))) { + xfs_bmap_mark_sick(ip, XFS_DATA_FORK); return xfs_alert_fsblock_zero(ip, imap); + } if (imap->br_startblock == HOLESTARTBLOCK) { iomap->addr = IOMAP_NULL_ADDR; @@ -110,7 +116,9 @@ xfs_bmbt_to_iomap( iomap->addr = IOMAP_NULL_ADDR; iomap->type = IOMAP_DELALLOC; } else { - iomap->addr = BBTOB(xfs_fsb_to_db(ip, imap->br_startblock)); + xfs_daddr_t daddr = xfs_fsb_to_db(ip, imap->br_startblock); + + iomap->addr = BBTOB(daddr); if (mapping_flags & IOMAP_DAX) iomap->addr += target->bt_dax_part_off; @@ -119,6 +127,14 @@ xfs_bmbt_to_iomap( else iomap->type = IOMAP_MAPPED; + /* + * Mark iomaps starting at the first sector of a RTG as merge + * boundary so that each I/O completions is contained to a + * single RTG. + */ + if (XFS_IS_REALTIME_INODE(ip) && xfs_has_rtgroups(mp) && + xfs_rtbno_is_group_start(mp, imap->br_startblock)) + iomap->flags |= IOMAP_F_BOUNDARY; } iomap->offset = XFS_FSB_TO_B(mp, imap->br_startoff); iomap->length = XFS_FSB_TO_B(mp, imap->br_blockcount); @@ -294,9 +310,7 @@ xfs_iomap_write_direct( if (error) return error; - error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, nr_exts); - if (error == -EFBIG) - error = xfs_iext_count_upgrade(tp, ip, nr_exts); + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, nr_exts); if (error) goto out_trans_cancel; @@ -317,16 +331,10 @@ xfs_iomap_write_direct( if (error) goto out_unlock; - /* - * Copy any maps to caller's array and return any error. - */ - if (nimaps == 0) { - error = -ENOSPC; - goto out_unlock; - } - - if (unlikely(!xfs_valid_startblock(ip, imap->br_startblock))) + if (unlikely(!xfs_valid_startblock(ip, imap->br_startblock))) { + xfs_bmap_mark_sick(ip, XFS_DATA_FORK); error = xfs_alert_fsblock_zero(ip, imap); + } out_unlock: *seq = xfs_iomap_inode_sequence(ip, 0); @@ -345,16 +353,26 @@ xfs_quota_need_throttle( xfs_fsblock_t alloc_blocks) { struct xfs_dquot *dq = xfs_inode_dquot(ip, type); + struct xfs_dquot_res *res; + struct xfs_dquot_pre *pre; if (!dq || !xfs_this_quota_on(ip->i_mount, type)) return false; + if (XFS_IS_REALTIME_INODE(ip)) { + res = &dq->q_rtb; + pre = &dq->q_rtb_prealloc; + } else { + res = &dq->q_blk; + pre = &dq->q_blk_prealloc; + } + /* no hi watermark, no throttle */ - if (!dq->q_prealloc_hi_wmark) + if (!pre->q_prealloc_hi_wmark) return false; /* under the lo watermark, no throttle */ - if (dq->q_blk.reserved + alloc_blocks < dq->q_prealloc_lo_wmark) + if (res->reserved + alloc_blocks < pre->q_prealloc_lo_wmark) return false; return true; @@ -369,22 +387,35 @@ xfs_quota_calc_throttle( int64_t *qfreesp) { struct xfs_dquot *dq = xfs_inode_dquot(ip, type); + struct xfs_dquot_res *res; + struct xfs_dquot_pre *pre; int64_t freesp; int shift = 0; + if (!dq) { + res = NULL; + pre = NULL; + } else if (XFS_IS_REALTIME_INODE(ip)) { + res = &dq->q_rtb; + pre = &dq->q_rtb_prealloc; + } else { + res = &dq->q_blk; + pre = &dq->q_blk_prealloc; + } + /* no dq, or over hi wmark, squash the prealloc completely */ - if (!dq || dq->q_blk.reserved >= dq->q_prealloc_hi_wmark) { + if (!res || res->reserved >= pre->q_prealloc_hi_wmark) { *qblocks = 0; *qfreesp = 0; return; } - freesp = dq->q_prealloc_hi_wmark - dq->q_blk.reserved; - if (freesp < dq->q_low_space[XFS_QLOWSP_5_PCNT]) { + freesp = pre->q_prealloc_hi_wmark - res->reserved; + if (freesp < pre->q_low_space[XFS_QLOWSP_5_PCNT]) { shift = 2; - if (freesp < dq->q_low_space[XFS_QLOWSP_3_PCNT]) + if (freesp < pre->q_low_space[XFS_QLOWSP_3_PCNT]) shift += 2; - if (freesp < dq->q_low_space[XFS_QLOWSP_1_PCNT]) + if (freesp < pre->q_low_space[XFS_QLOWSP_1_PCNT]) shift += 2; } @@ -398,6 +429,29 @@ xfs_quota_calc_throttle( } } +static int64_t +xfs_iomap_freesp( + struct percpu_counter *counter, + uint64_t low_space[XFS_LOWSP_MAX], + int *shift) +{ + int64_t freesp; + + freesp = percpu_counter_read_positive(counter); + if (freesp < low_space[XFS_LOWSP_5_PCNT]) { + *shift = 2; + if (freesp < low_space[XFS_LOWSP_4_PCNT]) + (*shift)++; + if (freesp < low_space[XFS_LOWSP_3_PCNT]) + (*shift)++; + if (freesp < low_space[XFS_LOWSP_2_PCNT]) + (*shift)++; + if (freesp < low_space[XFS_LOWSP_1_PCNT]) + (*shift)++; + } + return freesp; +} + /* * If we don't have a user specified preallocation size, dynamically increase * the preallocation size as the size of the file grows. Cap the maximum size @@ -480,18 +534,13 @@ xfs_iomap_prealloc_size( alloc_blocks = XFS_FILEOFF_MIN(roundup_pow_of_two(XFS_MAX_BMBT_EXTLEN), alloc_blocks); - freesp = percpu_counter_read_positive(&mp->m_fdblocks); - if (freesp < mp->m_low_space[XFS_LOWSP_5_PCNT]) { - shift = 2; - if (freesp < mp->m_low_space[XFS_LOWSP_4_PCNT]) - shift++; - if (freesp < mp->m_low_space[XFS_LOWSP_3_PCNT]) - shift++; - if (freesp < mp->m_low_space[XFS_LOWSP_2_PCNT]) - shift++; - if (freesp < mp->m_low_space[XFS_LOWSP_1_PCNT]) - shift++; - } + if (unlikely(XFS_IS_REALTIME_INODE(ip))) + freesp = xfs_rtbxlen_to_blen(mp, + xfs_iomap_freesp(&mp->m_frextents, + mp->m_low_rtexts, &shift)); + else + freesp = xfs_iomap_freesp(&mp->m_fdblocks, mp->m_low_space, + &shift); /* * Check each quota to cap the prealloc size, provide a shift value to @@ -600,11 +649,8 @@ xfs_iomap_write_unwritten( if (error) return error; - error = xfs_iext_count_may_overflow(ip, XFS_DATA_FORK, + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, XFS_IEXT_WRITE_UNWRITTEN_CNT); - if (error == -EFBIG) - error = xfs_iext_count_upgrade(tp, ip, - XFS_IEXT_WRITE_UNWRITTEN_CNT); if (error) goto error_on_bmapi_transaction; @@ -639,8 +685,10 @@ xfs_iomap_write_unwritten( if (error) return error; - if (unlikely(!xfs_valid_startblock(ip, imap.br_startblock))) + if (unlikely(!xfs_valid_startblock(ip, imap.br_startblock))) { + xfs_bmap_mark_sick(ip, XFS_DATA_FORK); return xfs_alert_fsblock_zero(ip, &imap); + } if ((numblks_fsb = imap.br_blockcount) == 0) { /* @@ -693,7 +741,7 @@ imap_needs_cow( return false; /* when zeroing we don't have to COW holes or unwritten extents */ - if (flags & IOMAP_ZERO) { + if (flags & (IOMAP_UNSHARE | IOMAP_ZERO)) { if (!nimaps || imap->br_startblock == HOLESTARTBLOCK || imap->br_state == XFS_EXT_UNWRITTEN) @@ -703,53 +751,30 @@ imap_needs_cow( return true; } +/* + * Extents not yet cached requires exclusive access, don't block for + * IOMAP_NOWAIT. + * + * This is basically an opencoded xfs_ilock_data_map_shared() call, but with + * support for IOMAP_NOWAIT. + */ static int xfs_ilock_for_iomap( struct xfs_inode *ip, unsigned flags, unsigned *lockmode) { - unsigned int mode = *lockmode; - bool is_write = flags & (IOMAP_WRITE | IOMAP_ZERO); - - /* - * COW writes may allocate delalloc space or convert unwritten COW - * extents, so we need to make sure to take the lock exclusively here. - */ - if (xfs_is_cow_inode(ip) && is_write) - mode = XFS_ILOCK_EXCL; - - /* - * Extents not yet cached requires exclusive access, don't block. This - * is an opencoded xfs_ilock_data_map_shared() call but with - * non-blocking behaviour. - */ - if (xfs_need_iread_extents(&ip->i_df)) { - if (flags & IOMAP_NOWAIT) - return -EAGAIN; - mode = XFS_ILOCK_EXCL; - } - -relock: if (flags & IOMAP_NOWAIT) { - if (!xfs_ilock_nowait(ip, mode)) + if (xfs_need_iread_extents(&ip->i_df)) + return -EAGAIN; + if (!xfs_ilock_nowait(ip, *lockmode)) return -EAGAIN; } else { - xfs_ilock(ip, mode); + if (xfs_need_iread_extents(&ip->i_df)) + *lockmode = XFS_ILOCK_EXCL; + xfs_ilock(ip, *lockmode); } - /* - * The reflink iflag could have changed since the earlier unlocked - * check, so if we got ILOCK_SHARED for a write and but we're now a - * reflink inode we have to switch to ILOCK_EXCL and relock. - */ - if (mode == XFS_ILOCK_SHARED && is_write && xfs_is_cow_inode(ip)) { - xfs_iunlock(ip, mode); - mode = XFS_ILOCK_EXCL; - goto relock; - } - - *lockmode = mode; return 0; } @@ -787,7 +812,7 @@ xfs_direct_write_iomap_begin( int nimaps = 1, error = 0; bool shared = false; u16 iomap_flags = 0; - unsigned int lockmode = XFS_ILOCK_SHARED; + unsigned int lockmode; u64 seq; ASSERT(flags & (IOMAP_WRITE | IOMAP_ZERO)); @@ -803,10 +828,30 @@ xfs_direct_write_iomap_begin( if (offset + length > i_size_read(inode)) iomap_flags |= IOMAP_F_DIRTY; + /* + * COW writes may allocate delalloc space or convert unwritten COW + * extents, so we need to make sure to take the lock exclusively here. + */ + if (xfs_is_cow_inode(ip)) + lockmode = XFS_ILOCK_EXCL; + else + lockmode = XFS_ILOCK_SHARED; + +relock: error = xfs_ilock_for_iomap(ip, flags, &lockmode); if (error) return error; + /* + * The reflink iflag could have changed since the earlier unlocked + * check, check if it again and relock if needed. + */ + if (xfs_is_cow_inode(ip) && lockmode == XFS_ILOCK_SHARED) { + xfs_iunlock(ip, lockmode); + lockmode = XFS_ILOCK_EXCL; + goto relock; + } + error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap, &nimaps, 0); if (error) @@ -931,10 +976,8 @@ xfs_dax_write_iomap_end( if (!xfs_is_cow_inode(ip)) return 0; - if (!written) { - xfs_reflink_cancel_cow_range(ip, pos, length, true); - return 0; - } + if (!written) + return xfs_reflink_cancel_cow_range(ip, pos, length, true); return xfs_reflink_end_cow(ip, pos, written); } @@ -964,6 +1007,7 @@ xfs_buffered_write_iomap_begin( int allocfork = XFS_DATA_FORK; int error = 0; unsigned int lockmode = XFS_ILOCK_EXCL; + unsigned int iomap_flags = 0; u64 seq; if (xfs_is_shutdown(mp)) @@ -974,8 +1018,6 @@ xfs_buffered_write_iomap_begin( return xfs_direct_write_iomap_begin(inode, offset, count, flags, iomap, srcmap); - ASSERT(!XFS_IS_REALTIME_INODE(ip)); - error = xfs_qm_dqattach(ip); if (error) return error; @@ -986,6 +1028,7 @@ xfs_buffered_write_iomap_begin( if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(&ip->i_df)) || XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) { + xfs_bmap_mark_sick(ip, XFS_DATA_FORK); error = -EFSCORRUPTED; goto out_unlock; } @@ -1014,6 +1057,24 @@ xfs_buffered_write_iomap_begin( } /* + * For zeroing, trim a delalloc extent that extends beyond the EOF + * block. If it starts beyond the EOF block, convert it to an + * unwritten extent. + */ + if ((flags & IOMAP_ZERO) && imap.br_startoff <= offset_fsb && + isnullstartblock(imap.br_startblock)) { + xfs_fileoff_t eof_fsb = XFS_B_TO_FSB(mp, XFS_ISIZE(ip)); + + if (offset_fsb >= eof_fsb) + goto convert_delay; + if (end_fsb > eof_fsb) { + end_fsb = eof_fsb; + xfs_trim_extent(&imap, offset_fsb, + end_fsb - offset_fsb); + } + } + + /* * Search the COW fork extent list even if we did not find a data fork * extent. This serves two purposes: first this implements the * speculative preallocation using cowextsize, so that we also unshare @@ -1117,76 +1178,77 @@ xfs_buffered_write_iomap_begin( } } -retry: - error = xfs_bmapi_reserve_delalloc(ip, allocfork, offset_fsb, - end_fsb - offset_fsb, prealloc_blocks, - allocfork == XFS_DATA_FORK ? &imap : &cmap, - allocfork == XFS_DATA_FORK ? &icur : &ccur, - allocfork == XFS_DATA_FORK ? eof : cow_eof); - switch (error) { - case 0: - break; - case -ENOSPC: - case -EDQUOT: - /* retry without any preallocation */ - trace_xfs_delalloc_enospc(ip, offset, count); - if (prealloc_blocks) { - prealloc_blocks = 0; - goto retry; - } - fallthrough; - default: - goto out_unlock; - } - + /* + * Flag newly allocated delalloc blocks with IOMAP_F_NEW so we punch + * them out if the write happens to fail. + */ + iomap_flags |= IOMAP_F_NEW; if (allocfork == XFS_COW_FORK) { + error = xfs_bmapi_reserve_delalloc(ip, allocfork, offset_fsb, + end_fsb - offset_fsb, prealloc_blocks, &cmap, + &ccur, cow_eof); + if (error) + goto out_unlock; + trace_xfs_iomap_alloc(ip, offset, count, allocfork, &cmap); goto found_cow; } - /* - * Flag newly allocated delalloc blocks with IOMAP_F_NEW so we punch - * them out if the write happens to fail. - */ - seq = xfs_iomap_inode_sequence(ip, IOMAP_F_NEW); - xfs_iunlock(ip, XFS_ILOCK_EXCL); - trace_xfs_iomap_alloc(ip, offset, count, allocfork, &imap); - return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, IOMAP_F_NEW, seq); + error = xfs_bmapi_reserve_delalloc(ip, allocfork, offset_fsb, + end_fsb - offset_fsb, prealloc_blocks, &imap, &icur, + eof); + if (error) + goto out_unlock; + trace_xfs_iomap_alloc(ip, offset, count, allocfork, &imap); found_imap: - seq = xfs_iomap_inode_sequence(ip, 0); - xfs_iunlock(ip, XFS_ILOCK_EXCL); - return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, 0, seq); + seq = xfs_iomap_inode_sequence(ip, iomap_flags); + xfs_iunlock(ip, lockmode); + return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, iomap_flags, seq); + +convert_delay: + xfs_iunlock(ip, lockmode); + truncate_pagecache(inode, offset); + error = xfs_bmapi_convert_delalloc(ip, XFS_DATA_FORK, offset, + iomap, NULL); + if (error) + return error; + + trace_xfs_iomap_alloc(ip, offset, count, XFS_DATA_FORK, &imap); + return 0; found_cow: - seq = xfs_iomap_inode_sequence(ip, 0); if (imap.br_startoff <= offset_fsb) { - error = xfs_bmbt_to_iomap(ip, srcmap, &imap, flags, 0, seq); + error = xfs_bmbt_to_iomap(ip, srcmap, &imap, flags, 0, + xfs_iomap_inode_sequence(ip, 0)); if (error) goto out_unlock; - seq = xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED); - xfs_iunlock(ip, XFS_ILOCK_EXCL); - return xfs_bmbt_to_iomap(ip, iomap, &cmap, flags, - IOMAP_F_SHARED, seq); + } else { + xfs_trim_extent(&cmap, offset_fsb, + imap.br_startoff - offset_fsb); } - xfs_trim_extent(&cmap, offset_fsb, imap.br_startoff - offset_fsb); - xfs_iunlock(ip, XFS_ILOCK_EXCL); - return xfs_bmbt_to_iomap(ip, iomap, &cmap, flags, 0, seq); + iomap_flags |= IOMAP_F_SHARED; + seq = xfs_iomap_inode_sequence(ip, iomap_flags); + xfs_iunlock(ip, lockmode); + return xfs_bmbt_to_iomap(ip, iomap, &cmap, flags, iomap_flags, seq); out_unlock: - xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_iunlock(ip, lockmode); return error; } -static int +static void xfs_buffered_write_delalloc_punch( struct inode *inode, loff_t offset, - loff_t length) + loff_t length, + struct iomap *iomap) { - return xfs_bmap_punch_delalloc_range(XFS_I(inode), offset, - offset + length); + xfs_bmap_punch_delalloc_range(XFS_I(inode), + (iomap->flags & IOMAP_F_SHARED) ? + XFS_COW_FORK : XFS_DATA_FORK, + offset, offset + length); } static int @@ -1198,17 +1260,38 @@ xfs_buffered_write_iomap_end( unsigned flags, struct iomap *iomap) { + loff_t start_byte, end_byte; - struct xfs_mount *mp = XFS_M(inode->i_sb); - int error; + /* If we didn't reserve the blocks, we're not allowed to punch them. */ + if (iomap->type != IOMAP_DELALLOC || !(iomap->flags & IOMAP_F_NEW)) + return 0; - error = iomap_file_buffered_write_punch_delalloc(inode, iomap, offset, - length, written, &xfs_buffered_write_delalloc_punch); - if (error && !xfs_is_shutdown(mp)) { - xfs_alert(mp, "%s: unable to clean up ino 0x%llx", - __func__, XFS_I(inode)->i_ino); - return error; + /* + * iomap_page_mkwrite() will never fail in a way that requires delalloc + * extents that it allocated to be revoked. Hence never try to release + * them here. + */ + if (flags & IOMAP_FAULT) + return 0; + + /* Nothing to do if we've written the entire delalloc extent */ + start_byte = iomap_last_written_block(inode, offset, written); + end_byte = round_up(offset + length, i_blocksize(inode)); + if (start_byte >= end_byte) + return 0; + + /* For zeroing operations the callers already hold invalidate_lock. */ + if (flags & (IOMAP_UNSHARE | IOMAP_ZERO)) { + rwsem_assert_held_write(&inode->i_mapping->invalidate_lock); + iomap_write_delalloc_release(inode, start_byte, end_byte, flags, + iomap, xfs_buffered_write_delalloc_punch); + } else { + filemap_invalidate_lock(inode->i_mapping); + iomap_write_delalloc_release(inode, start_byte, end_byte, flags, + iomap, xfs_buffered_write_delalloc_punch); + filemap_invalidate_unlock(inode->i_mapping); } + return 0; } @@ -1217,15 +1300,6 @@ const struct iomap_ops xfs_buffered_write_iomap_ops = { .iomap_end = xfs_buffered_write_iomap_end, }; -/* - * iomap_page_mkwrite() will never fail in a way that requires delalloc extents - * that it allocated to be revoked. Hence we do not need an .iomap_end method - * for this operation. - */ -const struct iomap_ops xfs_page_mkwrite_iomap_ops = { - .iomap_begin = xfs_buffered_write_iomap_begin, -}; - static int xfs_read_iomap_begin( struct inode *inode, @@ -1323,7 +1397,7 @@ xfs_seek_iomap_begin( if (cow_fsb != NULLFILEOFF && cow_fsb <= offset_fsb) { if (data_fsb < cow_fsb + cmap.br_blockcount) end_fsb = min(end_fsb, data_fsb); - xfs_trim_extent(&cmap, offset_fsb, end_fsb); + xfs_trim_extent(&cmap, offset_fsb, end_fsb - offset_fsb); seq = xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED); error = xfs_bmbt_to_iomap(ip, iomap, &cmap, flags, IOMAP_F_SHARED, seq); @@ -1348,7 +1422,7 @@ xfs_seek_iomap_begin( imap.br_state = XFS_EXT_NORM; done: seq = xfs_iomap_inode_sequence(ip, 0); - xfs_trim_extent(&imap, offset_fsb, end_fsb); + xfs_trim_extent(&imap, offset_fsb, end_fsb - offset_fsb); error = xfs_bmbt_to_iomap(ip, iomap, &imap, flags, 0, seq); out_unlock: xfs_iunlock(ip, lockmode); @@ -1415,6 +1489,8 @@ xfs_zero_range( { struct inode *inode = VFS_I(ip); + xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL); + if (IS_DAX(inode)) return dax_zero_range(inode, pos, len, did_zero, &xfs_dax_write_iomap_ops); |