diff options
Diffstat (limited to 'fs/ntfs3/inode.c')
| -rw-r--r-- | fs/ntfs3/inode.c | 774 |
1 files changed, 463 insertions, 311 deletions
diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c index a87ab3ad3cd3..0a9ac5efeb67 100644 --- a/fs/ntfs3/inode.c +++ b/fs/ntfs3/inode.c @@ -18,7 +18,7 @@ #include "ntfs_fs.h" /* - * ntfs_read_mft - Read record and parses MFT. + * ntfs_read_mft - Read record and parse MFT. */ static struct inode *ntfs_read_mft(struct inode *inode, const struct cpu_str *name, @@ -37,13 +37,14 @@ static struct inode *ntfs_read_mft(struct inode *inode, bool is_dir; unsigned long ino = inode->i_ino; u32 rp_fa = 0, asize, t32; - u16 roff, rsize, names = 0; + u16 roff, rsize, names = 0, links = 0; const struct ATTR_FILE_NAME *fname = NULL; const struct INDEX_ROOT *root; struct REPARSE_DATA_BUFFER rp; // 0x18 bytes u64 t64; struct MFT_REC *rec; struct runs_tree *run; + struct timespec64 ts; inode->i_op = NULL; /* Setup 'uid' and 'gid' */ @@ -81,7 +82,7 @@ static struct inode *ntfs_read_mft(struct inode *inode, le16_to_cpu(ref->seq), le16_to_cpu(rec->seq)); goto out; } else if (!is_rec_inuse(rec)) { - err = -EINVAL; + err = -ESTALE; ntfs_err(sb, "Inode r=%x is not in use!", (u32)ino); goto out; } @@ -92,12 +93,20 @@ static struct inode *ntfs_read_mft(struct inode *inode, goto out; } - if (!is_rec_base(rec)) - goto Ok; + if (!is_rec_base(rec)) { + err = -EINVAL; + goto out; + } /* Record should contain $I30 root. */ is_dir = rec->flags & RECORD_FLAG_DIR; + /* MFT_REC_MFT is not a dir */ + if (is_dir && ino == MFT_REC_MFT) { + err = -EINVAL; + goto out; + } + inode->i_generation = le16_to_cpu(rec->seq); /* Enumerate all struct Attributes MFT. */ @@ -129,6 +138,22 @@ next_attr: rsize = attr->non_res ? 0 : le32_to_cpu(attr->res.data_size); asize = le32_to_cpu(attr->size); + /* + * Really this check was done in 'ni_enum_attr_ex' -> ... 'mi_enum_attr'. + * There not critical to check this case again + */ + if (attr->name_len && + sizeof(short) * attr->name_len + le16_to_cpu(attr->name_off) > + asize) + goto out; + + if (attr->non_res) { + t64 = le64_to_cpu(attr->nres.alloc_size); + if (le64_to_cpu(attr->nres.data_size) > t64 || + le64_to_cpu(attr->nres.valid_size) > t64) + goto out; + } + switch (attr->type) { case ATTR_STD: if (attr->non_res || @@ -144,9 +169,12 @@ next_attr: #ifdef STATX_BTIME nt2kernel(std5->cr_time, &ni->i_crtime); #endif - nt2kernel(std5->a_time, &inode->i_atime); - nt2kernel(std5->c_time, &inode->i_ctime); - nt2kernel(std5->m_time, &inode->i_mtime); + nt2kernel(std5->a_time, &ts); + inode_set_atime_to_ts(inode, ts); + nt2kernel(std5->c_time, &ts); + inode_set_ctime_to_ts(inode, ts); + nt2kernel(std5->m_time, &ts); + inode_set_mtime_to_ts(inode, ts); ni->std_fa = std5->fa; @@ -172,11 +200,12 @@ next_attr: rsize < SIZEOF_ATTRIBUTE_FILENAME) goto out; + names += 1; fname = Add2Ptr(attr, roff); if (fname->type == FILE_NAME_DOS) goto next_attr; - names += 1; + links += 1; if (name && name->len == fname->name_len && !ntfs_cmp_names_cpu(name, (struct le_str *)&fname->name_len, NULL, false)) @@ -238,8 +267,8 @@ next_attr: if (!attr->nres.alloc_size) goto next_attr; - run = ino == MFT_REC_BITMAP ? &sbi->used.bitmap.run - : &ni->file.run; + run = ino == MFT_REC_BITMAP ? &sbi->used.bitmap.run : + &ni->file.run; break; case ATTR_ROOT: @@ -247,7 +276,6 @@ next_attr: goto out; root = Add2Ptr(attr, roff); - is_root = true; if (attr->name_len != ARRAY_SIZE(I30_NAME) || memcmp(attr_name(attr), I30_NAME, sizeof(I30_NAME))) @@ -260,15 +288,16 @@ next_attr: if (!is_dir) goto next_attr; + is_root = true; ni->ni_flags |= NI_FLAG_DIR; err = indx_init(&ni->dir, sbi, attr, INDEX_MUTEX_I30); if (err) goto out; - mode = sb->s_root - ? (S_IFDIR | (0777 & sbi->options->fs_dmask_inv)) - : (S_IFDIR | 0777); + mode = sb->s_root ? + (S_IFDIR | (0777 & sbi->options->fs_dmask_inv)) : + (S_IFDIR | 0777); goto next_attr; case ATTR_ALLOC: @@ -317,9 +346,7 @@ next_attr: inode->i_size = le16_to_cpu(rp.SymbolicLinkReparseBuffer .PrintNameLength) / sizeof(u16); - ni->i_valid = inode->i_size; - /* Clear directory bit. */ if (ni->ni_flags & NI_FLAG_DIR) { indx_clear(&ni->dir); @@ -364,7 +391,13 @@ next_attr: attr_unpack_run: roff = le16_to_cpu(attr->nres.run_off); + if (roff > asize) { + err = -EINVAL; + goto out; + } + t64 = le64_to_cpu(attr->nres.svcn); + err = run_unpack_ex(run, sbi, ino, t64, le64_to_cpu(attr->nres.evcn), t64, Add2Ptr(attr, roff), asize - roff); if (err < 0) @@ -377,8 +410,10 @@ end_enum: if (!std5) goto out; + if (is_bad_inode(inode)) + goto out; + if (!is_match && name) { - /* Reuse rec as buffer for ascii name. */ err = -ENOENT; goto out; } @@ -393,11 +428,12 @@ end_enum: if (names != le16_to_cpu(rec->hard_links)) { /* Correct minor error on the fly. Do not mark inode as dirty. */ + ntfs_inode_warn(inode, "Correct links count -> %u.", names); rec->hard_links = cpu_to_le16(names); ni->mi.dirty = true; } - set_nlink(inode, names); + set_nlink(inode, links); if (S_ISDIR(mode)) { ni->std_fa |= FILE_ATTRIBUTE_DIRECTORY; @@ -408,7 +444,9 @@ end_enum: * Usually a hard links to directories are disabled. */ inode->i_op = &ntfs_dir_inode_operations; - inode->i_fop = &ntfs_dir_operations; + inode->i_fop = unlikely(is_legacy_ntfs(sb)) ? + &ntfs_legacy_dir_operations : + &ntfs_dir_operations; ni->i_valid = 0; } else if (S_ISLNK(mode)) { ni->std_fa &= ~FILE_ATTRIBUTE_DIRECTORY; @@ -418,9 +456,11 @@ end_enum: } else if (S_ISREG(mode)) { ni->std_fa &= ~FILE_ATTRIBUTE_DIRECTORY; inode->i_op = &ntfs_file_inode_operations; - inode->i_fop = &ntfs_file_operations; - inode->i_mapping->a_ops = - is_compressed(ni) ? &ntfs_aops_cmpr : &ntfs_aops; + inode->i_fop = unlikely(is_legacy_ntfs(sb)) ? + &ntfs_legacy_file_operations : + &ntfs_file_operations; + inode->i_mapping->a_ops = is_compressed(ni) ? &ntfs_aops_cmpr : + &ntfs_aops; if (ino != MFT_REC_MFT) init_rwsem(&ni->file.run_lock); } else if (S_ISCHR(mode) || S_ISBLK(mode) || S_ISFIFO(mode) || @@ -430,6 +470,9 @@ end_enum: } else if (fname && fname->home.low == cpu_to_le32(MFT_REC_EXTEND) && fname->home.seq == cpu_to_le16(MFT_REC_EXTEND)) { /* Records in $Extend are not a files or general directories. */ + inode->i_op = &ntfs_file_inode_operations; + mode = S_IFREG; + init_rwsem(&ni->file.run_lock); } else { err = -EINVAL; goto out; @@ -449,7 +492,6 @@ end_enum: inode->i_flags |= S_NOSEC; } -Ok: if (ino == MFT_REC_MFT && !sb->s_root) sbi->mft.ni = NULL; @@ -496,13 +538,20 @@ struct inode *ntfs_iget5(struct super_block *sb, const struct MFT_REF *ref, return ERR_PTR(-ENOMEM); /* If this is a freshly allocated inode, need to read it now. */ - if (inode->i_state & I_NEW) + if (inode_state_read_once(inode) & I_NEW) inode = ntfs_read_mft(inode, name, ref); else if (ref->seq != ntfs_i(inode)->mi.mrec->seq) { - /* Inode overlaps? */ - make_bad_inode(inode); + /* + * Sequence number is not expected. + * Looks like inode was reused but caller uses the old reference + */ + iput(inode); + inode = ERR_PTR(-ESTALE); } + if (IS_ERR(inode)) + ntfs_set_state(sb->s_fs_info, NTFS_DIRTY_ERROR); + return inode; } @@ -521,7 +570,7 @@ static noinline int ntfs_get_block_vbo(struct inode *inode, u64 vbo, struct super_block *sb = inode->i_sb; struct ntfs_sb_info *sbi = sb->s_fs_info; struct ntfs_inode *ni = ntfs_i(inode); - struct page *page = bh->b_page; + struct folio *folio = bh->b_folio; u8 cluster_bits = sbi->cluster_bits; u32 block_size = sb->s_blocksize; u64 bytes, lbo, valid; @@ -534,25 +583,20 @@ static noinline int ntfs_get_block_vbo(struct inode *inode, u64 vbo, clear_buffer_new(bh); clear_buffer_uptodate(bh); - /* Direct write uses 'create=0'. */ - if (!create && vbo >= ni->i_valid) { - /* Out of valid. */ - return 0; - } - - if (vbo >= inode->i_size) { - /* Out of size. */ - return 0; - } - if (is_resident(ni)) { - ni_lock(ni); - err = attr_data_read_resident(ni, page); - ni_unlock(ni); - - if (!err) - set_buffer_uptodate(bh); + bh->b_blocknr = RESIDENT_LCN; bh->b_size = block_size; + if (!folio) { + /* direct io (read) or bmap call */ + err = 0; + } else { + ni_lock(ni); + err = attr_data_read_resident(ni, folio); + ni_unlock(ni); + + if (!err) + set_buffer_uptodate(bh); + } return err; } @@ -560,7 +604,8 @@ static noinline int ntfs_get_block_vbo(struct inode *inode, u64 vbo, off = vbo & sbi->cluster_mask; new = false; - err = attr_data_get_block(ni, vcn, 1, &lcn, &len, create ? &new : NULL); + err = attr_data_get_block(ni, vcn, 1, &lcn, &len, create ? &new : NULL, + create && sbi->cluster_size > PAGE_SIZE); if (err) goto out; @@ -569,7 +614,8 @@ static noinline int ntfs_get_block_vbo(struct inode *inode, u64 vbo, bytes = ((u64)len << cluster_bits) - off; - if (lcn == SPARSE_LCN) { + if (lcn >= sbi->used.bitmap.nbits) { + /* This case includes resident/compressed/sparse. */ if (!create) { if (bh->b_size > bytes) bh->b_size = bytes; @@ -578,11 +624,8 @@ static noinline int ntfs_get_block_vbo(struct inode *inode, u64 vbo, WARN_ON(1); } - if (new) { + if (new) set_buffer_new(bh); - if ((len << cluster_bits) > block_size) - ntfs_sparse_cluster(inode, page, vcn, len); - } lbo = ((u64)lcn << cluster_bits) + off; @@ -610,7 +653,6 @@ static noinline int ntfs_get_block_vbo(struct inode *inode, u64 vbo, } } else if (vbo >= valid) { /* Read out of valid data. */ - /* Should never be here 'cause already checked. */ clear_buffer_mapped(bh); } else if (vbo + bytes <= valid) { /* Normal read. */ @@ -623,19 +665,18 @@ static noinline int ntfs_get_block_vbo(struct inode *inode, u64 vbo, */ bytes = block_size; - if (page) { + if (folio) { u32 voff = valid - vbo; bh->b_size = block_size; off = vbo & (PAGE_SIZE - 1); - set_bh_page(bh, page, off); - ll_rw_block(REQ_OP_READ, 0, 1, &bh); - wait_on_buffer(bh); - if (!buffer_uptodate(bh)) { + folio_set_bh(bh, folio, off); + + if (bh_read(bh, 0) < 0) { err = -EIO; goto out; } - zero_user_segment(page, off + voff, off + block_size); + folio_zero_segment(folio, off + voff, off + block_size); } } @@ -676,32 +717,32 @@ static sector_t ntfs_bmap(struct address_space *mapping, sector_t block) return generic_block_bmap(mapping, block, ntfs_get_block_bmap); } -static int ntfs_readpage(struct file *file, struct page *page) +static int ntfs_read_folio(struct file *file, struct folio *folio) { int err; - struct address_space *mapping = page->mapping; + struct address_space *mapping = folio->mapping; struct inode *inode = mapping->host; struct ntfs_inode *ni = ntfs_i(inode); if (is_resident(ni)) { ni_lock(ni); - err = attr_data_read_resident(ni, page); + err = attr_data_read_resident(ni, folio); ni_unlock(ni); if (err != E_NTFS_NONRESIDENT) { - unlock_page(page); + folio_unlock(folio); return err; } } if (is_compressed(ni)) { ni_lock(ni); - err = ni_readpage_cmpr(ni, page); + err = ni_readpage_cmpr(ni, folio); ni_unlock(ni); return err; } /* Normal + sparse files. */ - return mpage_readpage(page, ntfs_get_block); + return mpage_read_folio(folio, ntfs_get_block); } static void ntfs_readahead(struct readahead_control *rac) @@ -757,6 +798,7 @@ static ssize_t ntfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) loff_t vbo = iocb->ki_pos; loff_t end; int wr = iov_iter_rw(iter) & WRITE; + size_t iter_count = iov_iter_count(iter); loff_t valid; ssize_t ret; @@ -765,15 +807,22 @@ static ssize_t ntfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) ret = 0; goto out; } + if (is_compressed(ni)) { + ret = 0; + goto out; + } ret = blockdev_direct_IO(iocb, inode, iter, - wr ? ntfs_get_block_direct_IO_W - : ntfs_get_block_direct_IO_R); - - if (ret <= 0) + wr ? ntfs_get_block_direct_IO_W : + ntfs_get_block_direct_IO_R); + + if (ret > 0) + end = vbo + ret; + else if (wr && ret == -EIOCBQUEUED) + end = vbo + iter_count; + else goto out; - end = vbo + ret; valid = ni->i_valid; if (wr) { if (end > valid && !S_ISBLK(inode->i_mode)) { @@ -823,35 +872,52 @@ out: return err; } -static int ntfs_writepage(struct page *page, struct writeback_control *wbc) +static int ntfs_resident_writepage(struct folio *folio, + struct writeback_control *wbc) { - struct address_space *mapping = page->mapping; + struct address_space *mapping = folio->mapping; struct inode *inode = mapping->host; struct ntfs_inode *ni = ntfs_i(inode); - int err; + int ret; - if (is_resident(ni)) { - ni_lock(ni); - err = attr_data_write_resident(ni, page); - ni_unlock(ni); - if (err != E_NTFS_NONRESIDENT) { - unlock_page(page); - return err; - } - } + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ni))) + return -EINVAL; + + if (unlikely(ntfs3_forced_shutdown(inode->i_sb))) + return -EIO; - return block_write_full_page(page, ntfs_get_block, wbc); + ni_lock(ni); + ret = attr_data_write_resident(ni, folio); + ni_unlock(ni); + + if (ret != E_NTFS_NONRESIDENT) + folio_unlock(folio); + mapping_set_error(mapping, ret); + return ret; } static int ntfs_writepages(struct address_space *mapping, struct writeback_control *wbc) { struct inode *inode = mapping->host; - struct ntfs_inode *ni = ntfs_i(inode); - /* Redirect call to 'ntfs_writepage' for resident files. */ - get_block_t *get_block = is_resident(ni) ? NULL : &ntfs_get_block; - return mpage_writepages(mapping, wbc, get_block); + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ntfs_i(inode)))) + return -EINVAL; + + if (unlikely(ntfs3_forced_shutdown(inode->i_sb))) + return -EIO; + + if (is_resident(ntfs_i(inode))) { + struct folio *folio = NULL; + int error; + + while ((folio = writeback_iter(mapping, wbc, folio, &error))) + error = ntfs_resident_writepage(folio, wbc); + return error; + } + return mpage_writepages(mapping, wbc, ntfs_get_block); } static int ntfs_get_block_write_begin(struct inode *inode, sector_t vbn, @@ -861,40 +927,46 @@ static int ntfs_get_block_write_begin(struct inode *inode, sector_t vbn, bh_result, create, GET_BLOCK_WRITE_BEGIN); } -static int ntfs_write_begin(struct file *file, struct address_space *mapping, - loff_t pos, u32 len, u32 flags, struct page **pagep, - void **fsdata) +int ntfs_write_begin(const struct kiocb *iocb, struct address_space *mapping, + loff_t pos, u32 len, struct folio **foliop, void **fsdata) { int err; struct inode *inode = mapping->host; struct ntfs_inode *ni = ntfs_i(inode); - *pagep = NULL; + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ni))) + return -EINVAL; + + if (unlikely(ntfs3_forced_shutdown(inode->i_sb))) + return -EIO; + if (is_resident(ni)) { - struct page *page = grab_cache_page_write_begin( - mapping, pos >> PAGE_SHIFT, flags); + struct folio *folio = __filemap_get_folio( + mapping, pos >> PAGE_SHIFT, FGP_WRITEBEGIN, + mapping_gfp_mask(mapping)); - if (!page) { - err = -ENOMEM; + if (IS_ERR(folio)) { + err = PTR_ERR(folio); goto out; } ni_lock(ni); - err = attr_data_read_resident(ni, page); + err = attr_data_read_resident(ni, folio); ni_unlock(ni); if (!err) { - *pagep = page; + *foliop = folio; goto out; } - unlock_page(page); - put_page(page); + folio_unlock(folio); + folio_put(folio); if (err != E_NTFS_NONRESIDENT) goto out; } - err = block_write_begin(mapping, pos, len, flags, pagep, + err = block_write_begin(mapping, pos, len, foliop, ntfs_get_block_write_begin); out: @@ -904,10 +976,9 @@ out: /* * ntfs_write_end - Address_space_operations::write_end. */ -static int ntfs_write_end(struct file *file, struct address_space *mapping, - loff_t pos, u32 len, u32 copied, struct page *page, - void *fsdata) - +int ntfs_write_end(const struct kiocb *iocb, struct address_space *mapping, + loff_t pos, u32 len, u32 copied, struct folio *folio, + void *fsdata) { struct inode *inode = mapping->host; struct ntfs_inode *ni = ntfs_i(inode); @@ -917,34 +988,35 @@ static int ntfs_write_end(struct file *file, struct address_space *mapping, if (is_resident(ni)) { ni_lock(ni); - err = attr_data_write_resident(ni, page); + err = attr_data_write_resident(ni, folio); ni_unlock(ni); if (!err) { + struct buffer_head *head = folio_buffers(folio); dirty = true; - /* Clear any buffers in page. */ - if (page_has_buffers(page)) { - struct buffer_head *head, *bh; + /* Clear any buffers in folio. */ + if (head) { + struct buffer_head *bh = head; - bh = head = page_buffers(page); do { clear_buffer_dirty(bh); clear_buffer_mapped(bh); set_buffer_uptodate(bh); } while (head != (bh = bh->b_this_page)); } - SetPageUptodate(page); + folio_mark_uptodate(folio); err = copied; } - unlock_page(page); - put_page(page); + folio_unlock(folio); + folio_put(folio); } else { - err = generic_write_end(file, mapping, pos, len, copied, page, + err = generic_write_end(iocb, mapping, pos, len, copied, folio, fsdata); } if (err >= 0) { if (!(ni->std_fa & FILE_ATTRIBUTE_ARCHIVE)) { - inode->i_ctime = inode->i_mtime = current_time(inode); + inode_set_mtime_to_ts(inode, + inode_set_ctime_current(inode)); ni->std_fa |= FILE_ATTRIBUTE_ARCHIVE; dirty = true; } @@ -954,6 +1026,11 @@ static int ntfs_write_end(struct file *file, struct address_space *mapping, dirty = true; } + if (pos + err > inode->i_size) { + i_size_write(inode, pos + err); + dirty = true; + } + if (dirty) mark_inode_dirty(inode); } @@ -961,45 +1038,6 @@ static int ntfs_write_end(struct file *file, struct address_space *mapping, return err; } -int reset_log_file(struct inode *inode) -{ - int err; - loff_t pos = 0; - u32 log_size = inode->i_size; - struct address_space *mapping = inode->i_mapping; - - for (;;) { - u32 len; - void *kaddr; - struct page *page; - - len = pos + PAGE_SIZE > log_size ? (log_size - pos) : PAGE_SIZE; - - err = block_write_begin(mapping, pos, len, 0, &page, - ntfs_get_block_write_begin); - if (err) - goto out; - - kaddr = kmap_atomic(page); - memset(kaddr, -1, len); - kunmap_atomic(kaddr); - flush_dcache_page(page); - - err = block_write_end(NULL, mapping, pos, len, len, page, NULL); - if (err < 0) - goto out; - pos += len; - - if (pos >= log_size) - break; - balance_dirty_pages_ratelimited(mapping); - } -out: - mark_inode_dirty_sync(inode); - - return err; -} - int ntfs3_write_inode(struct inode *inode, struct writeback_control *wbc) { return _ni_write_inode(inode, wbc->sync_mode == WB_SYNC_ALL); @@ -1011,68 +1049,26 @@ int ntfs_sync_inode(struct inode *inode) } /* - * writeback_inode - Helper function for ntfs_flush_inodes(). - * - * This writes both the inode and the file data blocks, waiting - * for in flight data blocks before the start of the call. It - * does not wait for any io started during the call. - */ -static int writeback_inode(struct inode *inode) -{ - int ret = sync_inode_metadata(inode, 0); - - if (!ret) - ret = filemap_fdatawrite(inode->i_mapping); - return ret; -} - -/* - * ntfs_flush_inodes - * - * Write data and metadata corresponding to i1 and i2. The io is - * started but we do not wait for any of it to finish. - * - * filemap_flush() is used for the block device, so if there is a dirty - * page for a block already in flight, we will not wait and start the - * io over again. + * Helper function to read file. */ -int ntfs_flush_inodes(struct super_block *sb, struct inode *i1, - struct inode *i2) -{ - int ret = 0; - - if (i1) - ret = writeback_inode(i1); - if (!ret && i2) - ret = writeback_inode(i2); - if (!ret) - ret = sync_blockdev_nowait(sb->s_bdev); - return ret; -} - -int inode_write_data(struct inode *inode, const void *data, size_t bytes) +int inode_read_data(struct inode *inode, void *data, size_t bytes) { pgoff_t idx; + struct address_space *mapping = inode->i_mapping; - /* Write non resident data. */ for (idx = 0; bytes; idx++) { size_t op = bytes > PAGE_SIZE ? PAGE_SIZE : bytes; - struct page *page = ntfs_map_page(inode->i_mapping, idx); + struct page *page = read_mapping_page(mapping, idx, NULL); + void *kaddr; if (IS_ERR(page)) return PTR_ERR(page); - lock_page(page); - WARN_ON(!PageUptodate(page)); - ClearPageUptodate(page); - - memcpy(page_address(page), data, op); - - flush_dcache_page(page); - SetPageUptodate(page); - unlock_page(page); + kaddr = kmap_atomic(page); + memcpy(data, kaddr, op); + kunmap_atomic(kaddr); - ntfs_unmap_page(page); + put_page(page); bytes -= op; data = Add2Ptr(data, PAGE_SIZE); @@ -1086,10 +1082,10 @@ int inode_write_data(struct inode *inode, const void *data, size_t bytes) * Number of bytes for REPARSE_DATA_BUFFER(IO_REPARSE_TAG_SYMLINK) * for unicode string of @uni_len length. */ -static inline u32 ntfs_reparse_bytes(u32 uni_len) +static inline u32 ntfs_reparse_bytes(u32 uni_len, bool is_absolute) { /* Header + unicode string + decorated unicode string. */ - return sizeof(short) * (2 * uni_len + 4) + + return sizeof(short) * (2 * uni_len + (is_absolute ? 4 : 0)) + offsetof(struct REPARSE_DATA_BUFFER, SymbolicLinkReparseBuffer.PathBuffer); } @@ -1102,8 +1098,11 @@ ntfs_create_reparse_buffer(struct ntfs_sb_info *sbi, const char *symname, struct REPARSE_DATA_BUFFER *rp; __le16 *rp_name; typeof(rp->SymbolicLinkReparseBuffer) *rs; + bool is_absolute; + + is_absolute = symname[0] && symname[1] == ':'; - rp = kzalloc(ntfs_reparse_bytes(2 * size + 2), GFP_NOFS); + rp = kzalloc(ntfs_reparse_bytes(2 * size + 2, is_absolute), GFP_NOFS); if (!rp) return ERR_PTR(-ENOMEM); @@ -1118,7 +1117,7 @@ ntfs_create_reparse_buffer(struct ntfs_sb_info *sbi, const char *symname, goto out; /* err = the length of unicode name of symlink. */ - *nsize = ntfs_reparse_bytes(err); + *nsize = ntfs_reparse_bytes(err, is_absolute); if (*nsize > sbi->reparse.max_size) { err = -EFBIG; @@ -1138,24 +1137,28 @@ ntfs_create_reparse_buffer(struct ntfs_sb_info *sbi, const char *symname, /* PrintName + SubstituteName. */ rs->SubstituteNameOffset = cpu_to_le16(sizeof(short) * err); - rs->SubstituteNameLength = cpu_to_le16(sizeof(short) * err + 8); + rs->SubstituteNameLength = + cpu_to_le16(sizeof(short) * err + (is_absolute ? 8 : 0)); rs->PrintNameLength = rs->SubstituteNameOffset; /* * TODO: Use relative path if possible to allow Windows to * parse this path. - * 0-absolute path 1- relative path (SYMLINK_FLAG_RELATIVE). + * 0-absolute path, 1- relative path (SYMLINK_FLAG_RELATIVE). */ - rs->Flags = 0; - - memmove(rp_name + err + 4, rp_name, sizeof(short) * err); - - /* Decorate SubstituteName. */ - rp_name += err; - rp_name[0] = cpu_to_le16('\\'); - rp_name[1] = cpu_to_le16('?'); - rp_name[2] = cpu_to_le16('?'); - rp_name[3] = cpu_to_le16('\\'); + rs->Flags = cpu_to_le32(is_absolute ? 0 : SYMLINK_FLAG_RELATIVE); + + memmove(rp_name + err + (is_absolute ? 4 : 0), rp_name, + sizeof(short) * err); + + if (is_absolute) { + /* Decorate SubstituteName. */ + rp_name += err; + rp_name[0] = cpu_to_le16('\\'); + rp_name[1] = cpu_to_le16('?'); + rp_name[2] = cpu_to_le16('?'); + rp_name[3] = cpu_to_le16('\\'); + } return rp; out: @@ -1163,11 +1166,22 @@ out: return ERR_PTR(err); } -struct inode *ntfs_create_inode(struct user_namespace *mnt_userns, - struct inode *dir, struct dentry *dentry, - const struct cpu_str *uni, umode_t mode, - dev_t dev, const char *symname, u32 size, - struct ntfs_fnd *fnd) +/* + * ntfs_create_inode + * + * Helper function for: + * - ntfs_create + * - ntfs_mknod + * - ntfs_symlink + * - ntfs_mkdir + * - ntfs_atomic_open + * + * NOTE: if fnd != NULL (ntfs_atomic_open) then @dir is locked + */ +int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const struct cpu_str *uni, + umode_t mode, dev_t dev, const char *symname, u32 size, + struct ntfs_fnd *fnd) { int err; struct super_block *sb = dir->i_sb; @@ -1192,7 +1206,11 @@ struct inode *ntfs_create_inode(struct user_namespace *mnt_userns, struct REPARSE_DATA_BUFFER *rp = NULL; bool rp_inserted = false; - ni_lock_dir(dir_ni); + /* New file will be resident or non resident. */ + const bool new_file_resident = 1; + + if (!fnd) + ni_lock_dir(dir_ni); dir_root = indx_get_root(&dir_ni->dir, dir_ni, NULL, NULL); if (!dir_root) { @@ -1255,16 +1273,31 @@ struct inode *ntfs_create_inode(struct user_namespace *mnt_userns, fa = FILE_ATTRIBUTE_ARCHIVE; } + /* If option "hide_dot_files" then set hidden attribute for dot files. */ + if (sbi->options->hide_dot_files && name->name[0] == '.') + fa |= FILE_ATTRIBUTE_HIDDEN; + if (!(mode & 0222)) fa |= FILE_ATTRIBUTE_READONLY; /* Allocate PATH_MAX bytes. */ - new_de = __getname(); + new_de = kmem_cache_zalloc(names_cachep, GFP_KERNEL); if (!new_de) { err = -ENOMEM; goto out1; } + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(dir_ni))) { + err = -EINVAL; + goto out2; + } + + if (unlikely(ntfs3_forced_shutdown(sb))) { + err = -EIO; + goto out2; + } + /* Mark rw ntfs as dirty. it will be cleared at umount. */ ntfs_set_state(sbi, NTFS_DIRTY_DIRTY); @@ -1273,18 +1306,17 @@ struct inode *ntfs_create_inode(struct user_namespace *mnt_userns, if (err) goto out2; - ni = ntfs_new_inode(sbi, ino, fa & FILE_ATTRIBUTE_DIRECTORY); + ni = ntfs_new_inode(sbi, ino, S_ISDIR(mode) ? RECORD_FLAG_DIR : 0); if (IS_ERR(ni)) { err = PTR_ERR(ni); ni = NULL; goto out3; } inode = &ni->vfs_inode; - inode_init_owner(mnt_userns, inode, dir, mode); + inode_init_owner(idmap, inode, dir, mode); mode = inode->i_mode; - inode->i_atime = inode->i_mtime = inode->i_ctime = ni->i_crtime = - current_time(inode); + ni->i_crtime = current_time(inode); rec = ni->mi.mrec; rec->hard_links = cpu_to_le16(1); @@ -1325,10 +1357,9 @@ struct inode *ntfs_create_inode(struct user_namespace *mnt_userns, attr->res.data_size = cpu_to_le32(dsize); std5->cr_time = std5->m_time = std5->c_time = std5->a_time = - kernel2nt(&inode->i_atime); + kernel2nt(&ni->i_crtime); - ni->std_fa = fa; - std5->fa = fa; + std5->fa = ni->std_fa = fa; attr = Add2Ptr(attr, asize); @@ -1340,12 +1371,19 @@ struct inode *ntfs_create_inode(struct user_namespace *mnt_userns, mi_get_ref(&ni->mi, &new_de->ref); fname = (struct ATTR_FILE_NAME *)(new_de + 1); + + if (sbi->options->windows_names && + !valid_windows_name(sbi, (struct le_str *)&fname->name_len)) { + err = -EINVAL; + goto out4; + } + mi_get_ref(&dir_ni->mi, &fname->home); fname->dup.cr_time = fname->dup.m_time = fname->dup.c_time = fname->dup.a_time = std5->cr_time; fname->dup.alloc_size = fname->dup.data_size = 0; fname->dup.fa = std5->fa; - fname->dup.ea_size = fname->dup.reparse = 0; + fname->dup.extend_data = S_ISLNK(mode) ? IO_REPARSE_TAG_SYMLINK : 0; dsize = le16_to_cpu(new_de->key_size); asize = ALIGN(SIZEOF_RESIDENT + dsize, 8); @@ -1396,8 +1434,7 @@ struct inode *ntfs_create_inode(struct user_namespace *mnt_userns, root = Add2Ptr(attr, sizeof(I30_NAME) + SIZEOF_RESIDENT); memcpy(root, dir_root, offsetof(struct INDEX_ROOT, ihdr)); - root->ihdr.de_off = - cpu_to_le32(sizeof(struct INDEX_HDR)); // 0x10 + root->ihdr.de_off = cpu_to_le32(sizeof(struct INDEX_HDR)); root->ihdr.used = cpu_to_le32(sizeof(struct INDEX_HDR) + sizeof(struct NTFS_DE)); root->ihdr.total = root->ihdr.used; @@ -1417,7 +1454,7 @@ struct inode *ntfs_create_inode(struct user_namespace *mnt_userns, attr->size = cpu_to_le32(SIZEOF_RESIDENT); attr->name_off = SIZEOF_RESIDENT_LE; attr->res.data_off = SIZEOF_RESIDENT_LE; - } else if (S_ISREG(mode)) { + } else if (!new_file_resident && S_ISREG(mode)) { /* * Regular file. Create empty non resident data attribute. */ @@ -1433,7 +1470,7 @@ struct inode *ntfs_create_inode(struct user_namespace *mnt_userns, attr->size = cpu_to_le32(SIZEOF_NONRESIDENT_EX + 8); attr->name_off = SIZEOF_NONRESIDENT_EX_LE; attr->flags = ATTR_FLAG_COMPRESSED; - attr->nres.c_unit = COMPRESSION_UNIT; + attr->nres.c_unit = NTFS_LZNT_CUNIT; asize = SIZEOF_NONRESIDENT_EX + 8; } else { attr->size = cpu_to_le32(SIZEOF_NONRESIDENT + 8); @@ -1484,7 +1521,7 @@ struct inode *ntfs_create_inode(struct user_namespace *mnt_userns, /* * Below function 'ntfs_save_wsl_perm' requires 0x78 bytes. - * It is good idea to keep extened attributes resident. + * It is good idea to keep extended attributes resident. */ if (asize + t16 + 0x78 + 8 > sbi->record_size) { CLST alen; @@ -1503,8 +1540,8 @@ struct inode *ntfs_create_inode(struct user_namespace *mnt_userns, cpu_to_le64(ntfs_up_cluster(sbi, nsize)); err = attr_allocate_clusters(sbi, &ni->file.run, 0, 0, - clst, NULL, 0, &alen, 0, - NULL); + clst, NULL, ALLOCATE_DEF, + &alen, 0, NULL, NULL); if (err) goto out5; @@ -1520,11 +1557,15 @@ struct inode *ntfs_create_inode(struct user_namespace *mnt_userns, } asize = SIZEOF_NONRESIDENT + ALIGN(err, 8); + /* Write non resident data. */ + err = ntfs_sb_write_run(sbi, &ni->file.run, 0, rp, + nsize, 0); + if (err) + goto out5; } else { attr->res.data_off = SIZEOF_RESIDENT_LE; attr->res.data_size = cpu_to_le32(nsize); memcpy(Add2Ptr(attr, SIZEOF_RESIDENT), rp, nsize); - nsize = 0; } /* Size of symlink equals the length of input string. */ inode->i_size = size; @@ -1545,21 +1586,13 @@ struct inode *ntfs_create_inode(struct user_namespace *mnt_userns, rec->used = cpu_to_le32(PtrOffset(rec, attr) + 8); rec->next_attr_id = cpu_to_le16(aid); - /* Step 2: Add new name in index. */ - err = indx_insert_entry(&dir_ni->dir, dir_ni, new_de, sbi, fnd, 0); - if (err) - goto out6; - - /* Unlock parent directory before ntfs_init_acl. */ - ni_unlock(dir_ni); - inode->i_generation = le16_to_cpu(rec->seq); - dir->i_mtime = dir->i_ctime = inode->i_atime; - if (S_ISDIR(mode)) { inode->i_op = &ntfs_dir_inode_operations; - inode->i_fop = &ntfs_dir_operations; + inode->i_fop = unlikely(is_legacy_ntfs(sb)) ? + &ntfs_legacy_dir_operations : + &ntfs_dir_operations; } else if (S_ISLNK(mode)) { inode->i_op = &ntfs_link_inode_operations; inode->i_fop = NULL; @@ -1568,9 +1601,11 @@ struct inode *ntfs_create_inode(struct user_namespace *mnt_userns, inode_nohighmem(inode); } else if (S_ISREG(mode)) { inode->i_op = &ntfs_file_inode_operations; - inode->i_fop = &ntfs_file_operations; - inode->i_mapping->a_ops = - is_compressed(ni) ? &ntfs_aops_cmpr : &ntfs_aops; + inode->i_fop = unlikely(is_legacy_ntfs(sb)) ? + &ntfs_legacy_file_operations : + &ntfs_file_operations; + inode->i_mapping->a_ops = is_compressed(ni) ? &ntfs_aops_cmpr : + &ntfs_aops; init_rwsem(&ni->file.run_lock); } else { inode->i_op = &ntfs_special_inode_operations; @@ -1579,51 +1614,84 @@ struct inode *ntfs_create_inode(struct user_namespace *mnt_userns, #ifdef CONFIG_NTFS3_FS_POSIX_ACL if (!S_ISLNK(mode) && (sb->s_flags & SB_POSIXACL)) { - err = ntfs_init_acl(mnt_userns, inode, dir); + err = ntfs_init_acl(idmap, inode, dir); if (err) - goto out7; + goto out5; } else #endif { inode->i_flags |= S_NOSEC; } - /* Write non resident data. */ - if (nsize) { - err = ntfs_sb_write_run(sbi, &ni->file.run, 0, rp, nsize, 0); + if (!S_ISLNK(mode)) { + /* + * ntfs_init_acl and ntfs_save_wsl_perm update extended attribute. + * The packed size of extended attribute is stored in direntry too. + * 'fname' here points to inside new_de. + */ + err = ntfs_save_wsl_perm(inode, &fname->dup.extend_data); if (err) - goto out7; + goto out6; + + /* + * update ea_size in file_name attribute too. + * Use ni_find_attr cause layout of MFT record may be changed + * in ntfs_init_acl and ntfs_save_wsl_perm. + */ + attr = ni_find_attr(ni, NULL, NULL, ATTR_NAME, NULL, 0, NULL, + NULL); + if (attr) { + struct ATTR_FILE_NAME *fn; + + fn = resident_data_ex(attr, SIZEOF_ATTRIBUTE_FILENAME); + if (fn) + fn->dup.extend_data = fname->dup.extend_data; + } } + /* We do not need to update parent directory later */ + ni->ni_flags &= ~NI_FLAG_UPDATE_PARENT; + + /* Step 2: Add new name in index. */ + err = indx_insert_entry(&dir_ni->dir, dir_ni, new_de, sbi, fnd, 0); + if (err) + goto out6; + /* * Call 'd_instantiate' after inode->i_op is set * but before finish_open. */ d_instantiate(dentry, inode); - ntfs_save_wsl_perm(inode); + /* Set original time. inode times (i_ctime) may be changed in ntfs_init_acl. */ + inode_set_atime_to_ts(inode, ni->i_crtime); + inode_set_ctime_to_ts(inode, ni->i_crtime); + inode_set_mtime_to_ts(inode, ni->i_crtime); + inode_set_mtime_to_ts(dir, ni->i_crtime); + inode_set_ctime_to_ts(dir, ni->i_crtime); + mark_inode_dirty(dir); mark_inode_dirty(inode); /* Normal exit. */ goto out2; -out7: - - /* Undo 'indx_insert_entry'. */ - ni_lock_dir(dir_ni); - indx_delete_entry(&dir_ni->dir, dir_ni, new_de + 1, - le16_to_cpu(new_de->key_size), sbi); - /* ni_unlock(dir_ni); will be called later. */ out6: + attr = ni_find_attr(ni, NULL, NULL, ATTR_EA, NULL, 0, NULL, NULL); + if (attr && attr->non_res) { + /* Delete ATTR_EA, if non-resident. */ + struct runs_tree run; + run_init(&run); + attr_set_size(ni, ATTR_EA, NULL, 0, &run, 0, NULL, false, NULL); + run_close(&run); + } + if (rp_inserted) ntfs_remove_reparse(sbi, IO_REPARSE_TAG_SYMLINK, &new_de->ref); out5: - if (S_ISDIR(mode) || run_is_empty(&ni->file.run)) - goto out4; - - run_deallocate(sbi, &ni->file.run, false); + if (!S_ISDIR(mode)) + run_deallocate(sbi, &ni->file.run, false); out4: clear_rec_inuse(rec); @@ -1631,21 +1699,20 @@ out4: ni->mi.dirty = false; discard_new_inode(inode); out3: - ntfs_mark_rec_free(sbi, ino); + ntfs_mark_rec_free(sbi, ino, false); out2: __putname(new_de); kfree(rp); out1: - if (err) { + if (!fnd) ni_unlock(dir_ni); - return ERR_PTR(err); - } - unlock_new_inode(inode); + if (!err) + unlock_new_inode(inode); - return inode; + return err; } int ntfs_link_inode(struct inode *inode, struct dentry *dentry) @@ -1654,10 +1721,9 @@ int ntfs_link_inode(struct inode *inode, struct dentry *dentry) struct ntfs_inode *ni = ntfs_i(inode); struct ntfs_sb_info *sbi = inode->i_sb->s_fs_info; struct NTFS_DE *de; - struct ATTR_FILE_NAME *de_name; /* Allocate PATH_MAX bytes. */ - de = __getname(); + de = kmem_cache_zalloc(names_cachep, GFP_KERNEL); if (!de) return -ENOMEM; @@ -1669,15 +1735,6 @@ int ntfs_link_inode(struct inode *inode, struct dentry *dentry) if (err) goto out; - de_name = (struct ATTR_FILE_NAME *)(de + 1); - /* Fill duplicate info. */ - de_name->dup.cr_time = de_name->dup.m_time = de_name->dup.c_time = - de_name->dup.a_time = kernel2nt(&inode->i_ctime); - de_name->dup.alloc_size = de_name->dup.data_size = - cpu_to_le64(inode->i_size); - de_name->dup.fa = ni->std_fa; - de_name->dup.ea_size = de_name->dup.reparse = 0; - err = ni_add_name(ntfs_i(d_inode(dentry->d_parent)), ni, de); out: __putname(de); @@ -1704,7 +1761,7 @@ int ntfs_unlink_inode(struct inode *dir, const struct dentry *dentry) return -EINVAL; /* Allocate PATH_MAX bytes. */ - de = __getname(); + de = kmem_cache_zalloc(names_cachep, GFP_KERNEL); if (!de) return -ENOMEM; @@ -1724,15 +1781,13 @@ int ntfs_unlink_inode(struct inode *dir, const struct dentry *dentry) if (!err) { drop_nlink(inode); - dir->i_mtime = dir->i_ctime = current_time(dir); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); mark_inode_dirty(dir); - inode->i_ctime = dir->i_ctime; + inode_set_ctime_to_ts(inode, inode_get_ctime(dir)); if (inode->i_nlink) mark_inode_dirty(inode); } else if (!ni_remove_name_undo(dir_ni, ni, de, de2, undo_remove)) { - make_bad_inode(inode); - ntfs_inode_err(inode, "failed to undo unlink"); - ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + _ntfs_bad_inode(inode); } else { if (ni_is_dirty(dir)) mark_inode_dirty(dir); @@ -1750,16 +1805,109 @@ void ntfs_evict_inode(struct inode *inode) { truncate_inode_pages_final(&inode->i_data); - if (inode->i_nlink) - _ni_write_inode(inode, inode_needs_sync(inode)); - invalidate_inode_buffers(inode); clear_inode(inode); ni_clear(ntfs_i(inode)); } -static noinline int ntfs_readlink_hlp(struct inode *inode, char *buffer, +/* + * ntfs_translate_junction + * + * Translate a Windows junction target to the Linux equivalent. + * On junctions, targets are always absolute (they include the drive + * letter). We have no way of knowing if the target is for the current + * mounted device or not so we just assume it is. + */ +static int ntfs_translate_junction(const struct super_block *sb, + const struct dentry *link_de, char *target, + int target_len, int target_max) +{ + int tl_len, err = target_len; + char *link_path_buffer = NULL, *link_path; + char *translated = NULL; + char *target_start; + int copy_len; + + link_path_buffer = kmalloc(PATH_MAX, GFP_NOFS); + if (!link_path_buffer) { + err = -ENOMEM; + goto out; + } + /* Get link path, relative to mount point */ + link_path = dentry_path_raw(link_de, link_path_buffer, PATH_MAX); + if (IS_ERR(link_path)) { + ntfs_err(sb, "Error getting link path"); + err = -EINVAL; + goto out; + } + + translated = kmalloc(PATH_MAX, GFP_NOFS); + if (!translated) { + err = -ENOMEM; + goto out; + } + + /* Make translated path a relative path to mount point */ + strcpy(translated, "./"); + ++link_path; /* Skip leading / */ + for (tl_len = sizeof("./") - 1; *link_path; ++link_path) { + if (*link_path == '/') { + if (PATH_MAX - tl_len < sizeof("../")) { + ntfs_err(sb, + "Link path %s has too many components", + link_path); + err = -EINVAL; + goto out; + } + strcpy(translated + tl_len, "../"); + tl_len += sizeof("../") - 1; + } + } + + /* Skip drive letter */ + target_start = target; + while (*target_start && *target_start != ':') + ++target_start; + + if (!*target_start) { + ntfs_err(sb, "Link target (%s) missing drive separator", + target); + err = -EINVAL; + goto out; + } + + /* Skip drive separator and leading /, if exists */ + target_start += 1 + (target_start[1] == '/'); + copy_len = target_len - (target_start - target); + + if (PATH_MAX - tl_len <= copy_len) { + ntfs_err(sb, "Link target %s too large for buffer (%d <= %d)", + target_start, PATH_MAX - tl_len, copy_len); + err = -EINVAL; + goto out; + } + + /* translated path has a trailing / and target_start does not */ + strcpy(translated + tl_len, target_start); + tl_len += copy_len; + if (target_max <= tl_len) { + ntfs_err(sb, "Target path %s too large for buffer (%d <= %d)", + translated, target_max, tl_len); + err = -EINVAL; + goto out; + } + strcpy(target, translated); + err = tl_len; + +out: + kfree(link_path_buffer); + kfree(translated); + return err; +} + +static noinline int ntfs_readlink_hlp(const struct dentry *link_de, + struct inode *inode, char *buffer, int buflen) { int i, err = -EINVAL; @@ -1902,6 +2050,11 @@ static noinline int ntfs_readlink_hlp(struct inode *inode, char *buffer, /* Always set last zero. */ buffer[err] = 0; + + /* If this is a junction, translate the link target. */ + if (rp->ReparseTag == IO_REPARSE_TAG_MOUNT_POINT) + err = ntfs_translate_junction(sb, link_de, buffer, err, buflen); + out: kfree(to_free); return err; @@ -1920,7 +2073,7 @@ static const char *ntfs_get_link(struct dentry *de, struct inode *inode, if (!ret) return ERR_PTR(-ENOMEM); - err = ntfs_readlink_hlp(inode, ret, PAGE_SIZE); + err = ntfs_readlink_hlp(de, inode, ret, PAGE_SIZE); if (err < 0) { kfree(ret); return ERR_PTR(err); @@ -1934,27 +2087,26 @@ static const char *ntfs_get_link(struct dentry *de, struct inode *inode, // clang-format off const struct inode_operations ntfs_link_inode_operations = { .get_link = ntfs_get_link, - .setattr = ntfs3_setattr, + .setattr = ntfs_setattr, .listxattr = ntfs_listxattr, - .permission = ntfs_permission, - .get_acl = ntfs_get_acl, - .set_acl = ntfs_set_acl, }; const struct address_space_operations ntfs_aops = { - .readpage = ntfs_readpage, + .read_folio = ntfs_read_folio, .readahead = ntfs_readahead, - .writepage = ntfs_writepage, .writepages = ntfs_writepages, .write_begin = ntfs_write_begin, .write_end = ntfs_write_end, .direct_IO = ntfs_direct_IO, .bmap = ntfs_bmap, - .set_page_dirty = __set_page_dirty_buffers, + .dirty_folio = block_dirty_folio, + .migrate_folio = buffer_migrate_folio, + .invalidate_folio = block_invalidate_folio, }; const struct address_space_operations ntfs_aops_cmpr = { - .readpage = ntfs_readpage, - .readahead = ntfs_readahead, + .read_folio = ntfs_read_folio, + .dirty_folio = block_dirty_folio, + .direct_IO = ntfs_direct_IO, }; // clang-format on |
