From 4dae666a6286ce5d9f4890f8b88af646473686f0 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 29 Apr 2020 02:13:57 +0200 Subject: btrfs: use the token::eb for all set/get helpers The token stores a copy of the extent buffer pointer but does not make any use of it besides sanity checks. We can use it and drop the eb parameter from several functions, this patch only switches the use inside the set/get helpers. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'fs/btrfs/struct-funcs.c') diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index 73f7987143df..7928d310f698 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -62,12 +62,12 @@ u##bits btrfs_get_token_##bits(const struct extent_buffer *eb, \ res = get_unaligned_le##bits(p + off); \ return res; \ } \ - err = map_private_extent_buffer(eb, offset, size, \ + err = map_private_extent_buffer(token->eb, offset, size, \ &kaddr, &map_start, &map_len); \ if (err) { \ __le##bits leres; \ \ - read_extent_buffer(eb, &leres, offset, size); \ + read_extent_buffer(token->eb, &leres, offset, size); \ return le##bits##_to_cpu(leres); \ } \ p = kaddr + part_offset - map_start; \ @@ -125,13 +125,13 @@ void btrfs_set_token_##bits(struct extent_buffer *eb, \ put_unaligned_le##bits(val, p + off); \ return; \ } \ - err = map_private_extent_buffer(eb, offset, size, \ + err = map_private_extent_buffer(token->eb, offset, size, \ &kaddr, &map_start, &map_len); \ if (err) { \ __le##bits val2; \ \ val2 = cpu_to_le##bits(val); \ - write_extent_buffer(eb, &val2, offset, size); \ + write_extent_buffer(token->eb, &val2, offset, size); \ return; \ } \ p = kaddr + part_offset - map_start; \ -- cgit From cc4c13d55cba8a0b81bc18243eabc57be1aa44d2 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 29 Apr 2020 02:15:56 +0200 Subject: btrfs: drop eb parameter from set/get token helpers Now that all set/get helpers use the eb from the token, we don't need to pass it to many btrfs_token_*/btrfs_set_token_* helpers, saving some stack space. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'fs/btrfs/struct-funcs.c') diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index 7928d310f698..cebd0b5e4f37 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -38,9 +38,8 @@ static inline void put_unaligned_le8(u8 val, void *p) */ #define DEFINE_BTRFS_SETGET_BITS(bits) \ -u##bits btrfs_get_token_##bits(const struct extent_buffer *eb, \ - const void *ptr, unsigned long off, \ - struct btrfs_map_token *token) \ +u##bits btrfs_get_token_##bits(struct btrfs_map_token *token, \ + const void *ptr, unsigned long off) \ { \ unsigned long part_offset = (unsigned long)ptr; \ unsigned long offset = part_offset + off; \ @@ -53,7 +52,6 @@ u##bits btrfs_get_token_##bits(const struct extent_buffer *eb, \ u##bits res; \ \ ASSERT(token); \ - ASSERT(token->eb == eb); \ \ if (token->kaddr && token->offset <= offset && \ (token->offset + PAGE_SIZE >= offset + size)) { \ @@ -101,10 +99,9 @@ u##bits btrfs_get_##bits(const struct extent_buffer *eb, \ res = get_unaligned_le##bits(p + off); \ return res; \ } \ -void btrfs_set_token_##bits(struct extent_buffer *eb, \ +void btrfs_set_token_##bits(struct btrfs_map_token *token, \ const void *ptr, unsigned long off, \ - u##bits val, \ - struct btrfs_map_token *token) \ + u##bits val) \ { \ unsigned long part_offset = (unsigned long)ptr; \ unsigned long offset = part_offset + off; \ @@ -116,7 +113,6 @@ void btrfs_set_token_##bits(struct extent_buffer *eb, \ int size = sizeof(u##bits); \ \ ASSERT(token); \ - ASSERT(token->eb == eb); \ \ if (token->kaddr && token->offset <= offset && \ (token->offset + PAGE_SIZE >= offset + size)) { \ -- cgit From 870b388db02cac33eebe40a1cbeb056cf13e9f40 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 29 Apr 2020 19:29:04 +0200 Subject: btrfs: preset set/get token with first page and drop condition All the set/get helpers first check if the token contains a cached address. After first use the address is always valid, but the extra check is done for each call. The token initialization can optimistically set it to the first extent buffer page, that we know always exists. Then the condition in all btrfs_token_*/btrfs_set_token_* can be simplified by removing the address check from the condition, but for development the assertion still makes sure it's valid. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'fs/btrfs/struct-funcs.c') diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index cebd0b5e4f37..cef628a5a9e0 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -52,8 +52,8 @@ u##bits btrfs_get_token_##bits(struct btrfs_map_token *token, \ u##bits res; \ \ ASSERT(token); \ - \ - if (token->kaddr && token->offset <= offset && \ + ASSERT(token->kaddr); \ + if (token->offset <= offset && \ (token->offset + PAGE_SIZE >= offset + size)) { \ kaddr = token->kaddr; \ p = kaddr + part_offset - token->offset; \ @@ -113,8 +113,8 @@ void btrfs_set_token_##bits(struct btrfs_map_token *token, \ int size = sizeof(u##bits); \ \ ASSERT(token); \ - \ - if (token->kaddr && token->offset <= offset && \ + ASSERT(token->kaddr); \ + if (token->offset <= offset && \ (token->offset + PAGE_SIZE >= offset + size)) { \ kaddr = token->kaddr; \ p = kaddr + part_offset - token->offset; \ -- cgit From 5e3946890c184a17b4d74baec89c96dce518f002 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Thu, 30 Apr 2020 23:38:11 +0200 Subject: btrfs: add separate bounds checker for set/get helpers The bounds checking is now done in map_private_extent_buffer but that will be removed in following patches and some sanity checks should still be done. There are two separate checks to see the kind of out of bounds access: partial (start offset is in the buffer) or complete (both start and end are out). Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) (limited to 'fs/btrfs/struct-funcs.c') diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index cef628a5a9e0..68c02997e60d 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -17,6 +17,27 @@ static inline void put_unaligned_le8(u8 val, void *p) *(u8 *)p = val; } +static bool check_setget_bounds(const struct extent_buffer *eb, + const void *ptr, unsigned off, int size) +{ + const unsigned long member_offset = (unsigned long)ptr + off; + + if (member_offset > eb->len) { + btrfs_warn(eb->fs_info, + "bad eb member start: ptr 0x%lx start %llu member offset %lu size %d", + (unsigned long)ptr, eb->start, member_offset, size); + return false; + } + if (member_offset + size > eb->len) { + btrfs_warn(eb->fs_info, + "bad eb member end: ptr 0x%lx start %llu member offset %lu size %d", + (unsigned long)ptr, eb->start, member_offset, size); + return false; + } + + return true; +} + /* * this is some deeply nasty code. * @@ -53,6 +74,7 @@ u##bits btrfs_get_token_##bits(struct btrfs_map_token *token, \ \ ASSERT(token); \ ASSERT(token->kaddr); \ + ASSERT(check_setget_bounds(token->eb, ptr, off, size)); \ if (token->offset <= offset && \ (token->offset + PAGE_SIZE >= offset + size)) { \ kaddr = token->kaddr; \ @@ -87,6 +109,7 @@ u##bits btrfs_get_##bits(const struct extent_buffer *eb, \ int size = sizeof(u##bits); \ u##bits res; \ \ + ASSERT(check_setget_bounds(eb, ptr, off, size)); \ err = map_private_extent_buffer(eb, offset, size, \ &kaddr, &map_start, &map_len); \ if (err) { \ @@ -114,6 +137,7 @@ void btrfs_set_token_##bits(struct btrfs_map_token *token, \ \ ASSERT(token); \ ASSERT(token->kaddr); \ + ASSERT(check_setget_bounds(token->eb, ptr, off, size)); \ if (token->offset <= offset && \ (token->offset + PAGE_SIZE >= offset + size)) { \ kaddr = token->kaddr; \ @@ -147,6 +171,7 @@ void btrfs_set_##bits(struct extent_buffer *eb, void *ptr, \ unsigned long map_len; \ int size = sizeof(u##bits); \ \ + ASSERT(check_setget_bounds(eb, ptr, off, size)); \ err = map_private_extent_buffer(eb, offset, size, \ &kaddr, &map_start, &map_len); \ if (err) { \ -- cgit From 1441ed9b7a0f53f47afd0b5ccdcb447e50559165 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 29 Apr 2020 16:04:44 +0200 Subject: btrfs: speed up btrfs_get_##bits helpers The helpers unconditionally call map_private_extent_buffer to get the address of page containing the requested offset plus the mapping start and length. Depending on the return value, the fast path uses unaligned read to get data within a page, or fall back to read_extent_buffer that can handle reads spanning more pages. This is all wasteful. We know the number of bytes to read, 1/2/4/8 and can find out the page. Then simply check if it's contained or the fallback is needed. This saves one function call to map_private_extent_buffer and several unnecessary temporary variables. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) (limited to 'fs/btrfs/struct-funcs.c') diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index 68c02997e60d..e6d2bd019444 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -99,28 +99,19 @@ u##bits btrfs_get_token_##bits(struct btrfs_map_token *token, \ u##bits btrfs_get_##bits(const struct extent_buffer *eb, \ const void *ptr, unsigned long off) \ { \ - unsigned long part_offset = (unsigned long)ptr; \ - unsigned long offset = part_offset + off; \ - void *p; \ - int err; \ - char *kaddr; \ - unsigned long map_start; \ - unsigned long map_len; \ - int size = sizeof(u##bits); \ - u##bits res; \ + const unsigned long member_offset = (unsigned long)ptr + off; \ + const unsigned long oip = offset_in_page(member_offset); \ + const int size = sizeof(u##bits); \ + __le##bits leres; \ \ ASSERT(check_setget_bounds(eb, ptr, off, size)); \ - err = map_private_extent_buffer(eb, offset, size, \ - &kaddr, &map_start, &map_len); \ - if (err) { \ - __le##bits leres; \ - \ - read_extent_buffer(eb, &leres, offset, size); \ - return le##bits##_to_cpu(leres); \ + if (oip + size <= PAGE_SIZE) { \ + const unsigned long idx = member_offset >> PAGE_SHIFT; \ + const char *kaddr = page_address(eb->pages[idx]); \ + return get_unaligned_le##bits(kaddr + oip); \ } \ - p = kaddr + part_offset - map_start; \ - res = get_unaligned_le##bits(p + off); \ - return res; \ + read_extent_buffer(eb, &leres, member_offset, size); \ + return le##bits##_to_cpu(leres); \ } \ void btrfs_set_token_##bits(struct btrfs_map_token *token, \ const void *ptr, unsigned long off, \ -- cgit From 8f9da810ee1fac6b587e3b871ee73696906bcba3 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 29 Apr 2020 17:45:33 +0200 Subject: btrfs: speed up btrfs_get_token_##bits helpers The set/get token helpers either use the cached address in the token or unconditionally call map_private_extent_buffer to get the address of page containing the requested offset plus the mapping start and length. Depending on the return value, the fast path uses unaligned read to get data within a page, or fall back to read_extent_buffer that can handle reads spanning more pages. This is all wasteful. We know the number of bytes to read, 1/2/4/8 and can find out the page. Then simply check if it's contained or the fallback is needed. The token address is updated to the page, or the on the next index, expecting that the next read will use that. This saves one function call to map_private_extent_buffer and several unnecessary temporary variables. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 43 ++++++++++++++++--------------------------- 1 file changed, 16 insertions(+), 27 deletions(-) (limited to 'fs/btrfs/struct-funcs.c') diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index e6d2bd019444..e357e0bab397 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -62,39 +62,28 @@ static bool check_setget_bounds(const struct extent_buffer *eb, u##bits btrfs_get_token_##bits(struct btrfs_map_token *token, \ const void *ptr, unsigned long off) \ { \ - unsigned long part_offset = (unsigned long)ptr; \ - unsigned long offset = part_offset + off; \ - void *p; \ - int err; \ - char *kaddr; \ - unsigned long map_start; \ - unsigned long map_len; \ - int size = sizeof(u##bits); \ - u##bits res; \ + const unsigned long member_offset = (unsigned long)ptr + off; \ + const unsigned long idx = member_offset >> PAGE_SHIFT; \ + const unsigned long oip = offset_in_page(member_offset); \ + const int size = sizeof(u##bits); \ + __le##bits leres; \ \ ASSERT(token); \ ASSERT(token->kaddr); \ ASSERT(check_setget_bounds(token->eb, ptr, off, size)); \ - if (token->offset <= offset && \ - (token->offset + PAGE_SIZE >= offset + size)) { \ - kaddr = token->kaddr; \ - p = kaddr + part_offset - token->offset; \ - res = get_unaligned_le##bits(p + off); \ - return res; \ + if (token->offset <= member_offset && \ + member_offset + size <= token->offset + PAGE_SIZE) { \ + return get_unaligned_le##bits(token->kaddr + oip); \ } \ - err = map_private_extent_buffer(token->eb, offset, size, \ - &kaddr, &map_start, &map_len); \ - if (err) { \ - __le##bits leres; \ - \ - read_extent_buffer(token->eb, &leres, offset, size); \ - return le##bits##_to_cpu(leres); \ + if (oip + size <= PAGE_SIZE) { \ + token->kaddr = page_address(token->eb->pages[idx]); \ + token->offset = idx << PAGE_SHIFT; \ + return get_unaligned_le##bits(token->kaddr + oip); \ } \ - p = kaddr + part_offset - map_start; \ - res = get_unaligned_le##bits(p + off); \ - token->kaddr = kaddr; \ - token->offset = map_start; \ - return res; \ + token->kaddr = page_address(token->eb->pages[idx + 1]); \ + token->offset = (idx + 1) << PAGE_SHIFT; \ + read_extent_buffer(token->eb, &leres, member_offset, size); \ + return le##bits##_to_cpu(leres); \ } \ u##bits btrfs_get_##bits(const struct extent_buffer *eb, \ const void *ptr, unsigned long off) \ -- cgit From 029e4a42a2b22f3c82e5b325820219963aa3f13b Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 29 Apr 2020 18:07:04 +0200 Subject: btrfs: speed up btrfs_set_##bits helpers The helpers unconditionally call map_private_extent_buffer to get the address of page containing the requested offset plus the mapping start and length. Depending on the return value, the fast path uses unaligned put to write data within a page, or fall back to write_extent_buffer that can handle writes spanning more pages. This is all wasteful. We know the number of bytes to write, 1/2/4/8 and can find out the page. Then simply check if it's contained or the fallback is needed. This saves one function call to map_private_extent_buffer and several unnecessary temporary variables. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) (limited to 'fs/btrfs/struct-funcs.c') diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index e357e0bab397..f8a0357d10fd 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -142,27 +142,20 @@ void btrfs_set_token_##bits(struct btrfs_map_token *token, \ void btrfs_set_##bits(struct extent_buffer *eb, void *ptr, \ unsigned long off, u##bits val) \ { \ - unsigned long part_offset = (unsigned long)ptr; \ - unsigned long offset = part_offset + off; \ - void *p; \ - int err; \ - char *kaddr; \ - unsigned long map_start; \ - unsigned long map_len; \ - int size = sizeof(u##bits); \ + const unsigned long member_offset = (unsigned long)ptr + off; \ + const unsigned long oip = offset_in_page(member_offset); \ + const int size = sizeof(u##bits); \ + __le##bits leres; \ \ ASSERT(check_setget_bounds(eb, ptr, off, size)); \ - err = map_private_extent_buffer(eb, offset, size, \ - &kaddr, &map_start, &map_len); \ - if (err) { \ - __le##bits val2; \ - \ - val2 = cpu_to_le##bits(val); \ - write_extent_buffer(eb, &val2, offset, size); \ + if (oip + size <= PAGE_SIZE) { \ + const unsigned long idx = member_offset >> PAGE_SHIFT; \ + char *kaddr = page_address(eb->pages[idx]); \ + put_unaligned_le##bits(val, kaddr + oip); \ return; \ } \ - p = kaddr + part_offset - map_start; \ - put_unaligned_le##bits(val, p + off); \ + leres = cpu_to_le##bits(val); \ + write_extent_buffer(eb, &leres, member_offset, size); \ } DEFINE_BTRFS_SETGET_BITS(8) -- cgit From ce7afe8782a5293edbf4bc2165ac4a866ed430db Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 29 Apr 2020 18:23:37 +0200 Subject: btrfs: speed up btrfs_set_token_##bits helpers The set/get token helpers either use the cached address in the token or unconditionally call map_private_extent_buffer to get the address of page containing the requested offset plus the mapping start and length. Depending on the return value, the fast path uses unaligned put to write data within a page, or fall back to write_extent_buffer that can handle writes spanning more pages. This is all wasteful. We know the number of bytes to write, 1/2/4/8 and can find out the page. Then simply check if it's contained or the fallback is needed. The token address is updated to the page, or the on the next index, expecting that the next write will use that. This saves one function call to map_private_extent_buffer and several unnecessary temporary variables. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 40 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 24 deletions(-) (limited to 'fs/btrfs/struct-funcs.c') diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index f8a0357d10fd..67dfd1287c3e 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -106,38 +106,30 @@ void btrfs_set_token_##bits(struct btrfs_map_token *token, \ const void *ptr, unsigned long off, \ u##bits val) \ { \ - unsigned long part_offset = (unsigned long)ptr; \ - unsigned long offset = part_offset + off; \ - void *p; \ - int err; \ - char *kaddr; \ - unsigned long map_start; \ - unsigned long map_len; \ - int size = sizeof(u##bits); \ + const unsigned long member_offset = (unsigned long)ptr + off; \ + const unsigned long idx = member_offset >> PAGE_SHIFT; \ + const unsigned long oip = offset_in_page(member_offset); \ + const int size = sizeof(u##bits); \ + __le##bits leres; \ \ ASSERT(token); \ ASSERT(token->kaddr); \ ASSERT(check_setget_bounds(token->eb, ptr, off, size)); \ - if (token->offset <= offset && \ - (token->offset + PAGE_SIZE >= offset + size)) { \ - kaddr = token->kaddr; \ - p = kaddr + part_offset - token->offset; \ - put_unaligned_le##bits(val, p + off); \ + if (token->offset <= member_offset && \ + member_offset + size <= token->offset + PAGE_SIZE) { \ + put_unaligned_le##bits(val, token->kaddr + oip); \ return; \ } \ - err = map_private_extent_buffer(token->eb, offset, size, \ - &kaddr, &map_start, &map_len); \ - if (err) { \ - __le##bits val2; \ - \ - val2 = cpu_to_le##bits(val); \ - write_extent_buffer(token->eb, &val2, offset, size); \ + if (oip + size <= PAGE_SIZE) { \ + token->kaddr = page_address(token->eb->pages[idx]); \ + token->offset = idx << PAGE_SHIFT; \ + put_unaligned_le##bits(val, token->kaddr + oip); \ return; \ } \ - p = kaddr + part_offset - map_start; \ - put_unaligned_le##bits(val, p + off); \ - token->kaddr = kaddr; \ - token->offset = map_start; \ + token->kaddr = page_address(token->eb->pages[idx + 1]); \ + token->offset = (idx + 1) << PAGE_SHIFT; \ + leres = cpu_to_le##bits(val); \ + write_extent_buffer(token->eb, &leres, member_offset, size); \ } \ void btrfs_set_##bits(struct extent_buffer *eb, void *ptr, \ unsigned long off, u##bits val) \ -- cgit From 2b48966a4da4bcb35f0883bc23dcaf63fcb8557f Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 29 Apr 2020 03:04:10 +0200 Subject: btrfs: constify extent_buffer in the API functions There are many helpers around extent buffers, found in extent_io.h and ctree.h. Most of them can be converted to take constified eb as there are no changes to the extent buffer structure itself but rather the pages. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs/btrfs/struct-funcs.c') diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index 67dfd1287c3e..0b23aa0a32d5 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -131,7 +131,7 @@ void btrfs_set_token_##bits(struct btrfs_map_token *token, \ leres = cpu_to_le##bits(val); \ write_extent_buffer(token->eb, &leres, member_offset, size); \ } \ -void btrfs_set_##bits(struct extent_buffer *eb, void *ptr, \ +void btrfs_set_##bits(const struct extent_buffer *eb, void *ptr, \ unsigned long off, u##bits val) \ { \ const unsigned long member_offset = (unsigned long)ptr + off; \ -- cgit From 84da071f3df5a9c0df0352d1c52b0c36ddb1c53f Mon Sep 17 00:00:00 2001 From: David Sterba Date: Thu, 30 Apr 2020 17:57:55 +0200 Subject: btrfs: optimize split page read in btrfs_get_##bits The helper read_extent_buffer is called to do read of the data spanning two extent buffer pages. As the size is known, we can do the read directly in two steps. This removes one function call and compiler can optimize memcpy as the sizes are known at compile time. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'fs/btrfs/struct-funcs.c') diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index 0b23aa0a32d5..46a7269bee07 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -90,17 +90,20 @@ u##bits btrfs_get_##bits(const struct extent_buffer *eb, \ { \ const unsigned long member_offset = (unsigned long)ptr + off; \ const unsigned long oip = offset_in_page(member_offset); \ + const unsigned long idx = member_offset >> PAGE_SHIFT; \ + char *kaddr = page_address(eb->pages[idx]); \ const int size = sizeof(u##bits); \ - __le##bits leres; \ + const int part = PAGE_SIZE - oip; \ + u8 lebytes[sizeof(u##bits)]; \ \ ASSERT(check_setget_bounds(eb, ptr, off, size)); \ - if (oip + size <= PAGE_SIZE) { \ - const unsigned long idx = member_offset >> PAGE_SHIFT; \ - const char *kaddr = page_address(eb->pages[idx]); \ + if (oip + size <= PAGE_SIZE) \ return get_unaligned_le##bits(kaddr + oip); \ - } \ - read_extent_buffer(eb, &leres, member_offset, size); \ - return le##bits##_to_cpu(leres); \ + \ + memcpy(lebytes, kaddr + oip, part); \ + kaddr = page_address(eb->pages[idx + 1]); \ + memcpy(lebytes + part, kaddr, size - part); \ + return get_unaligned_le##bits(lebytes); \ } \ void btrfs_set_token_##bits(struct btrfs_map_token *token, \ const void *ptr, unsigned long off, \ -- cgit From ba8a9a0537770df69d9dc38c11312c9b0f840cf2 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Thu, 30 Apr 2020 17:57:55 +0200 Subject: btrfs: optimize split page read in btrfs_get_token_##bits The fallback path calls helper read_extent_buffer to do read of the data spanning two extent buffer pages. As the size is known, we can do the read directly in two steps. This removes one function call and compiler can optimize memcpy as the sizes are known at compile time. The cached token address is set to the second page. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'fs/btrfs/struct-funcs.c') diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index 46a7269bee07..63cab91507f8 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -66,7 +66,8 @@ u##bits btrfs_get_token_##bits(struct btrfs_map_token *token, \ const unsigned long idx = member_offset >> PAGE_SHIFT; \ const unsigned long oip = offset_in_page(member_offset); \ const int size = sizeof(u##bits); \ - __le##bits leres; \ + u8 lebytes[sizeof(u##bits)]; \ + const int part = PAGE_SIZE - oip; \ \ ASSERT(token); \ ASSERT(token->kaddr); \ @@ -75,15 +76,16 @@ u##bits btrfs_get_token_##bits(struct btrfs_map_token *token, \ member_offset + size <= token->offset + PAGE_SIZE) { \ return get_unaligned_le##bits(token->kaddr + oip); \ } \ - if (oip + size <= PAGE_SIZE) { \ - token->kaddr = page_address(token->eb->pages[idx]); \ - token->offset = idx << PAGE_SHIFT; \ + token->kaddr = page_address(token->eb->pages[idx]); \ + token->offset = idx << PAGE_SHIFT; \ + if (oip + size <= PAGE_SIZE) \ return get_unaligned_le##bits(token->kaddr + oip); \ - } \ + \ + memcpy(lebytes, token->kaddr + oip, part); \ token->kaddr = page_address(token->eb->pages[idx + 1]); \ token->offset = (idx + 1) << PAGE_SHIFT; \ - read_extent_buffer(token->eb, &leres, member_offset, size); \ - return le##bits##_to_cpu(leres); \ + memcpy(lebytes + part, token->kaddr, size - part); \ + return get_unaligned_le##bits(lebytes); \ } \ u##bits btrfs_get_##bits(const struct extent_buffer *eb, \ const void *ptr, unsigned long off) \ -- cgit From f4ca8c51d12631f1297d093f767c3a5ce5b95aff Mon Sep 17 00:00:00 2001 From: David Sterba Date: Thu, 30 Apr 2020 17:57:55 +0200 Subject: btrfs: optimize split page write in btrfs_set_##bits The helper write_extent_buffer is called to do write of the data spanning two extent buffer pages. As the size is known, we can do the write directly in two steps. This removes one function call and compiler can optimize memcpy as the sizes are known at compile time. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'fs/btrfs/struct-funcs.c') diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index 63cab91507f8..7987d3910660 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -141,18 +141,22 @@ void btrfs_set_##bits(const struct extent_buffer *eb, void *ptr, \ { \ const unsigned long member_offset = (unsigned long)ptr + off; \ const unsigned long oip = offset_in_page(member_offset); \ + const unsigned long idx = member_offset >> PAGE_SHIFT; \ + char *kaddr = page_address(eb->pages[idx]); \ const int size = sizeof(u##bits); \ - __le##bits leres; \ + const int part = PAGE_SIZE - oip; \ + u8 lebytes[sizeof(u##bits)]; \ \ ASSERT(check_setget_bounds(eb, ptr, off, size)); \ if (oip + size <= PAGE_SIZE) { \ - const unsigned long idx = member_offset >> PAGE_SHIFT; \ - char *kaddr = page_address(eb->pages[idx]); \ put_unaligned_le##bits(val, kaddr + oip); \ return; \ } \ - leres = cpu_to_le##bits(val); \ - write_extent_buffer(eb, &leres, member_offset, size); \ + \ + put_unaligned_le##bits(val, lebytes); \ + memcpy(kaddr + oip, lebytes, part); \ + kaddr = page_address(eb->pages[idx + 1]); \ + memcpy(kaddr, lebytes + part, size - part); \ } DEFINE_BTRFS_SETGET_BITS(8) -- cgit From f472d3c28385397b1a2891c4ec264b90cd6d2187 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Thu, 30 Apr 2020 17:57:55 +0200 Subject: btrfs: optimize split page write in btrfs_set_token_##bits The fallback path calls helper write_extent_buffer to do write of the data spanning two extent buffer pages. As the size is known, we can do the write directly in two steps. This removes one function call and compiler can optimize memcpy as the sizes are known at compile time. The cached token address is set to the second page. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'fs/btrfs/struct-funcs.c') diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index 7987d3910660..225ef6d7e949 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -115,7 +115,8 @@ void btrfs_set_token_##bits(struct btrfs_map_token *token, \ const unsigned long idx = member_offset >> PAGE_SHIFT; \ const unsigned long oip = offset_in_page(member_offset); \ const int size = sizeof(u##bits); \ - __le##bits leres; \ + u8 lebytes[sizeof(u##bits)]; \ + const int part = PAGE_SIZE - oip; \ \ ASSERT(token); \ ASSERT(token->kaddr); \ @@ -125,16 +126,17 @@ void btrfs_set_token_##bits(struct btrfs_map_token *token, \ put_unaligned_le##bits(val, token->kaddr + oip); \ return; \ } \ + token->kaddr = page_address(token->eb->pages[idx]); \ + token->offset = idx << PAGE_SHIFT; \ if (oip + size <= PAGE_SIZE) { \ - token->kaddr = page_address(token->eb->pages[idx]); \ - token->offset = idx << PAGE_SHIFT; \ put_unaligned_le##bits(val, token->kaddr + oip); \ return; \ } \ + put_unaligned_le##bits(val, lebytes); \ + memcpy(token->kaddr + oip, lebytes, part); \ token->kaddr = page_address(token->eb->pages[idx + 1]); \ token->offset = (idx + 1) << PAGE_SHIFT; \ - leres = cpu_to_le##bits(val); \ - write_extent_buffer(token->eb, &leres, member_offset, size); \ + memcpy(token->kaddr, lebytes + part, size - part); \ } \ void btrfs_set_##bits(const struct extent_buffer *eb, void *ptr, \ unsigned long off, u##bits val) \ -- cgit From 583e4a2384db843fa8ae9a67aff8ea0eed140e61 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 6 May 2020 20:54:13 +0200 Subject: btrfs: update documentation of set/get helpers Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) (limited to 'fs/btrfs/struct-funcs.c') diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index 225ef6d7e949..079b059818e9 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -39,23 +39,27 @@ static bool check_setget_bounds(const struct extent_buffer *eb, } /* - * this is some deeply nasty code. + * Macro templates that define helpers to read/write extent buffer data of a + * given size, that are also used via ctree.h for access to item members by + * specialized helpers. * - * The end result is that anyone who #includes ctree.h gets a - * declaration for the btrfs_set_foo functions and btrfs_foo functions, - * which are wrappers of btrfs_set_token_#bits functions and - * btrfs_get_token_#bits functions, which are defined in this file. + * Generic helpers: + * - btrfs_set_8 (for 8/16/32/64) + * - btrfs_get_8 (for 8/16/32/64) * - * These setget functions do all the extent_buffer related mapping - * required to efficiently read and write specific fields in the extent - * buffers. Every pointer to metadata items in btrfs is really just - * an unsigned long offset into the extent buffer which has been - * cast to a specific type. This gives us all the gcc type checking. + * Generic helpers with a token (cached address of the most recently accessed + * page): + * - btrfs_set_token_8 (for 8/16/32/64) + * - btrfs_get_token_8 (for 8/16/32/64) * - * The extent buffer api is used to do the page spanning work required to - * have a metadata blocksize different from the page size. + * The set/get functions handle data spanning two pages transparently, in case + * metadata block size is larger than page. Every pointer to metadata items is + * an offset into the extent buffer page array, cast to a specific type. This + * gives us all the type checking. * - * There are 2 variants defined, one with a token pointer and one without. + * The extent buffer pages stored in the array pages do not form a contiguous + * phyusical range, but the API functions assume the linear offset to the range + * from 0 to metadata node size. */ #define DEFINE_BTRFS_SETGET_BITS(bits) \ -- cgit