From 71a3cdf80c0d12aadca6f9b0f5a43883649bdbea Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Wed, 12 Jan 2022 11:49:53 +0000 Subject: bpf/scripts: Raise an exception if the correct number of helpers are not generated Currently bpf_helper_defs.h and the bpf helpers man page are auto-generated using function documentation present in bpf.h. If the documentation for the helper is missing or doesn't follow a specific format for e.g. if a function is documented as: * long bpf_kallsyms_lookup_name( const char *name, int name_sz, int flags, u64 *res ) instead of * long bpf_kallsyms_lookup_name(const char *name, int name_sz, int flags, u64 *res) (notice the extra space at the start and end of function arguments) then that helper is not dumped in the auto-generated header and results in an invalid call during eBPF runtime, even if all the code specific to the helper is correct. This patch checks the number of functions documented within the header file with those present as part of #define __BPF_FUNC_MAPPER and raises an Exception if they don't match. It is not needed with the currently documented upstream functions, but can help in debugging when developing new helpers when there might be missing or misformatted documentation. Signed-off-by: Usama Arif Signed-off-by: Andrii Nakryiko Reviewed-by: Quentin Monnet Acked-by: Song Liu Link: https://lore.kernel.org/bpf/20220112114953.722380-1-usama.arif@bytedance.com --- scripts/bpf_doc.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) (limited to 'scripts/bpf_doc.py') diff --git a/scripts/bpf_doc.py b/scripts/bpf_doc.py index a6403ddf5de7..5cf8ae2e72bd 100755 --- a/scripts/bpf_doc.py +++ b/scripts/bpf_doc.py @@ -87,6 +87,8 @@ class HeaderParser(object): self.line = '' self.helpers = [] self.commands = [] + self.desc_unique_helpers = set() + self.define_unique_helpers = [] def parse_element(self): proto = self.parse_symbol() @@ -193,19 +195,42 @@ class HeaderParser(object): except NoSyscallCommandFound: break - def parse_helpers(self): + def parse_desc_helpers(self): self.seek_to('* Start of BPF helper function descriptions:', 'Could not find start of eBPF helper descriptions list') while True: try: helper = self.parse_helper() self.helpers.append(helper) + proto = helper.proto_break_down() + self.desc_unique_helpers.add(proto['name']) except NoHelperFound: break + def parse_define_helpers(self): + # Parse the number of FN(...) in #define __BPF_FUNC_MAPPER to compare + # later with the number of unique function names present in description. + # Note: seek_to(..) discards the first line below the target search text, + # resulting in FN(unspec) being skipped and not added to self.define_unique_helpers. + self.seek_to('#define __BPF_FUNC_MAPPER(FN)', + 'Could not find start of eBPF helper definition list') + # Searches for either one or more FN(\w+) defines or a backslash for newline + p = re.compile('\s*(FN\(\w+\))+|\\\\') + fn_defines_str = '' + while True: + capture = p.match(self.line) + if capture: + fn_defines_str += self.line + else: + break + self.line = self.reader.readline() + # Find the number of occurences of FN(\w+) + self.define_unique_helpers = re.findall('FN\(\w+\)', fn_defines_str) + def run(self): self.parse_syscall() - self.parse_helpers() + self.parse_desc_helpers() + self.parse_define_helpers() self.reader.close() ############################################################################### @@ -295,6 +320,25 @@ class PrinterRST(Printer): print('') +def helper_number_check(desc_unique_helpers, define_unique_helpers): + """ + Checks the number of functions documented within the header file + with those present as part of #define __BPF_FUNC_MAPPER and raise an + Exception if they don't match. + """ + nr_desc_unique_helpers = len(desc_unique_helpers) + nr_define_unique_helpers = len(define_unique_helpers) + if nr_desc_unique_helpers != nr_define_unique_helpers: + helper_exception = ''' +The number of unique helpers in description (%d) doesn\'t match the number of unique helpers defined in __BPF_FUNC_MAPPER (%d) +''' % (nr_desc_unique_helpers, nr_define_unique_helpers) + if nr_desc_unique_helpers < nr_define_unique_helpers: + # Function description is parsed until no helper is found (which can be due to + # misformatting). Hence, only print the first missing/misformatted function. + helper_exception += ''' +The description for %s is not present or formatted correctly. +''' % (define_unique_helpers[nr_desc_unique_helpers]) + raise Exception(helper_exception) class PrinterHelpersRST(PrinterRST): """ @@ -305,6 +349,7 @@ class PrinterHelpersRST(PrinterRST): """ def __init__(self, parser): self.elements = parser.helpers + helper_number_check(parser.desc_unique_helpers, parser.define_unique_helpers) def print_header(self): header = '''\ @@ -509,6 +554,7 @@ class PrinterHelpers(Printer): """ def __init__(self, parser): self.elements = parser.helpers + helper_number_check(parser.desc_unique_helpers, parser.define_unique_helpers) type_fwds = [ 'struct bpf_fib_lookup', -- cgit From f1f3f67fd8ed6f512955bbbc76b04e9dc33ddeb6 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Wed, 19 Jan 2022 11:44:41 +0000 Subject: bpf/scripts: Make description and returns section for helpers/syscalls mandatory This enforce a minimal formatting consistency for the documentation. The description and returns missing for a few helpers have also been added. Signed-off-by: Usama Arif Signed-off-by: Andrii Nakryiko Reviewed-by: Quentin Monnet Link: https://lore.kernel.org/bpf/20220119114442.1452088-2-usama.arif@bytedance.com --- scripts/bpf_doc.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) (limited to 'scripts/bpf_doc.py') diff --git a/scripts/bpf_doc.py b/scripts/bpf_doc.py index 5cf8ae2e72bd..20441e5d2d33 100755 --- a/scripts/bpf_doc.py +++ b/scripts/bpf_doc.py @@ -92,14 +92,14 @@ class HeaderParser(object): def parse_element(self): proto = self.parse_symbol() - desc = self.parse_desc() - ret = self.parse_ret() + desc = self.parse_desc(proto) + ret = self.parse_ret(proto) return APIElement(proto=proto, desc=desc, ret=ret) def parse_helper(self): proto = self.parse_proto() - desc = self.parse_desc() - ret = self.parse_ret() + desc = self.parse_desc(proto) + ret = self.parse_ret(proto) return Helper(proto=proto, desc=desc, ret=ret) def parse_symbol(self): @@ -129,16 +129,15 @@ class HeaderParser(object): self.line = self.reader.readline() return capture.group(1) - def parse_desc(self): + def parse_desc(self, proto): p = re.compile(' \* ?(?:\t| {5,8})Description$') capture = p.match(self.line) if not capture: - # Helper can have empty description and we might be parsing another - # attribute: return but do not consume. - return '' + raise Exception("No description section found for " + proto) # Description can be several lines, some of them possibly empty, and it # stops when another subsection title is met. desc = '' + desc_present = False while True: self.line = self.reader.readline() if self.line == ' *\n': @@ -147,21 +146,24 @@ class HeaderParser(object): p = re.compile(' \* ?(?:\t| {5,8})(?:\t| {8})(.*)') capture = p.match(self.line) if capture: + desc_present = True desc += capture.group(1) + '\n' else: break + + if not desc_present: + raise Exception("No description found for " + proto) return desc - def parse_ret(self): + def parse_ret(self, proto): p = re.compile(' \* ?(?:\t| {5,8})Return$') capture = p.match(self.line) if not capture: - # Helper can have empty retval and we might be parsing another - # attribute: return but do not consume. - return '' + raise Exception("No return section found for " + proto) # Return value description can be several lines, some of them possibly # empty, and it stops when another subsection title is met. ret = '' + ret_present = False while True: self.line = self.reader.readline() if self.line == ' *\n': @@ -170,9 +172,13 @@ class HeaderParser(object): p = re.compile(' \* ?(?:\t| {5,8})(?:\t| {8})(.*)') capture = p.match(self.line) if capture: + ret_present = True ret += capture.group(1) + '\n' else: break + + if not ret_present: + raise Exception("No return found for " + proto) return ret def seek_to(self, target, help_message): -- cgit From 0ba3929e5b3d3fda05d3b9c8d0d20a90a084c19e Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Wed, 19 Jan 2022 11:44:42 +0000 Subject: bpf/scripts: Raise an exception if the correct number of sycalls are not generated Currently the syscalls rst and subsequently man page are auto-generated using function documentation present in bpf.h. If the documentation for the syscall is missing or doesn't follow a specific format, then that syscall is not dumped in the auto-generated rst. This patch checks the number of syscalls documented within the header file with those present as part of the enum bpf_cmd and raises an Exception if they don't match. It is not needed with the currently documented upstream syscalls, but can help in debugging when developing new syscalls when there might be missing or misformatted documentation. The function helper_number_check is moved to the Printer parent class and renamed to elem_number_check as all the most derived children classes are using this function now. Signed-off-by: Usama Arif Signed-off-by: Andrii Nakryiko Reviewed-by: Quentin Monnet Link: https://lore.kernel.org/bpf/20220119114442.1452088-3-usama.arif@bytedance.com --- scripts/bpf_doc.py | 86 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 27 deletions(-) (limited to 'scripts/bpf_doc.py') diff --git a/scripts/bpf_doc.py b/scripts/bpf_doc.py index 20441e5d2d33..096625242475 100755 --- a/scripts/bpf_doc.py +++ b/scripts/bpf_doc.py @@ -89,6 +89,8 @@ class HeaderParser(object): self.commands = [] self.desc_unique_helpers = set() self.define_unique_helpers = [] + self.desc_syscalls = [] + self.enum_syscalls = [] def parse_element(self): proto = self.parse_symbol() @@ -103,7 +105,7 @@ class HeaderParser(object): return Helper(proto=proto, desc=desc, ret=ret) def parse_symbol(self): - p = re.compile(' \* ?(.+)$') + p = re.compile(' \* ?(BPF\w+)$') capture = p.match(self.line) if not capture: raise NoSyscallCommandFound @@ -181,26 +183,55 @@ class HeaderParser(object): raise Exception("No return found for " + proto) return ret - def seek_to(self, target, help_message): + def seek_to(self, target, help_message, discard_lines = 1): self.reader.seek(0) offset = self.reader.read().find(target) if offset == -1: raise Exception(help_message) self.reader.seek(offset) self.reader.readline() - self.reader.readline() + for _ in range(discard_lines): + self.reader.readline() self.line = self.reader.readline() - def parse_syscall(self): + def parse_desc_syscall(self): self.seek_to('* DOC: eBPF Syscall Commands', 'Could not find start of eBPF syscall descriptions list') while True: try: command = self.parse_element() self.commands.append(command) + self.desc_syscalls.append(command.proto) + except NoSyscallCommandFound: break + def parse_enum_syscall(self): + self.seek_to('enum bpf_cmd {', + 'Could not find start of bpf_cmd enum', 0) + # Searches for either one or more BPF\w+ enums + bpf_p = re.compile('\s*(BPF\w+)+') + # Searches for an enum entry assigned to another entry, + # for e.g. BPF_PROG_RUN = BPF_PROG_TEST_RUN, which is + # not documented hence should be skipped in check to + # determine if the right number of syscalls are documented + assign_p = re.compile('\s*(BPF\w+)\s*=\s*(BPF\w+)') + bpf_cmd_str = '' + while True: + capture = assign_p.match(self.line) + if capture: + # Skip line if an enum entry is assigned to another entry + self.line = self.reader.readline() + continue + capture = bpf_p.match(self.line) + if capture: + bpf_cmd_str += self.line + else: + break + self.line = self.reader.readline() + # Find the number of occurences of BPF\w+ + self.enum_syscalls = re.findall('(BPF\w+)+', bpf_cmd_str) + def parse_desc_helpers(self): self.seek_to('* Start of BPF helper function descriptions:', 'Could not find start of eBPF helper descriptions list') @@ -234,7 +265,8 @@ class HeaderParser(object): self.define_unique_helpers = re.findall('FN\(\w+\)', fn_defines_str) def run(self): - self.parse_syscall() + self.parse_desc_syscall() + self.parse_enum_syscall() self.parse_desc_helpers() self.parse_define_helpers() self.reader.close() @@ -266,6 +298,25 @@ class Printer(object): self.print_one(elem) self.print_footer() + def elem_number_check(self, desc_unique_elem, define_unique_elem, type, instance): + """ + Checks the number of helpers/syscalls documented within the header file + description with those defined as part of enum/macro and raise an + Exception if they don't match. + """ + nr_desc_unique_elem = len(desc_unique_elem) + nr_define_unique_elem = len(define_unique_elem) + if nr_desc_unique_elem != nr_define_unique_elem: + exception_msg = ''' +The number of unique %s in description (%d) doesn\'t match the number of unique %s defined in %s (%d) +''' % (type, nr_desc_unique_elem, type, instance, nr_define_unique_elem) + if nr_desc_unique_elem < nr_define_unique_elem: + # Function description is parsed until no helper is found (which can be due to + # misformatting). Hence, only print the first missing/misformatted helper/enum. + exception_msg += ''' +The description for %s is not present or formatted correctly. +''' % (define_unique_elem[nr_desc_unique_elem]) + raise Exception(exception_msg) class PrinterRST(Printer): """ @@ -326,26 +377,6 @@ class PrinterRST(Printer): print('') -def helper_number_check(desc_unique_helpers, define_unique_helpers): - """ - Checks the number of functions documented within the header file - with those present as part of #define __BPF_FUNC_MAPPER and raise an - Exception if they don't match. - """ - nr_desc_unique_helpers = len(desc_unique_helpers) - nr_define_unique_helpers = len(define_unique_helpers) - if nr_desc_unique_helpers != nr_define_unique_helpers: - helper_exception = ''' -The number of unique helpers in description (%d) doesn\'t match the number of unique helpers defined in __BPF_FUNC_MAPPER (%d) -''' % (nr_desc_unique_helpers, nr_define_unique_helpers) - if nr_desc_unique_helpers < nr_define_unique_helpers: - # Function description is parsed until no helper is found (which can be due to - # misformatting). Hence, only print the first missing/misformatted function. - helper_exception += ''' -The description for %s is not present or formatted correctly. -''' % (define_unique_helpers[nr_desc_unique_helpers]) - raise Exception(helper_exception) - class PrinterHelpersRST(PrinterRST): """ A printer for dumping collected information about helpers as a ReStructured @@ -355,7 +386,7 @@ class PrinterHelpersRST(PrinterRST): """ def __init__(self, parser): self.elements = parser.helpers - helper_number_check(parser.desc_unique_helpers, parser.define_unique_helpers) + self.elem_number_check(parser.desc_unique_helpers, parser.define_unique_helpers, 'helper', '__BPF_FUNC_MAPPER') def print_header(self): header = '''\ @@ -529,6 +560,7 @@ class PrinterSyscallRST(PrinterRST): """ def __init__(self, parser): self.elements = parser.commands + self.elem_number_check(parser.desc_syscalls, parser.enum_syscalls, 'syscall', 'bpf_cmd') def print_header(self): header = '''\ @@ -560,7 +592,7 @@ class PrinterHelpers(Printer): """ def __init__(self, parser): self.elements = parser.helpers - helper_number_check(parser.desc_unique_helpers, parser.define_unique_helpers) + self.elem_number_check(parser.desc_unique_helpers, parser.define_unique_helpers, 'helper', '__BPF_FUNC_MAPPER') type_fwds = [ 'struct bpf_fib_lookup', -- cgit From 3bc253c2e652cf5f12cd8c00d80d8ec55d67d1a7 Mon Sep 17 00:00:00 2001 From: Geliang Tang Date: Thu, 19 May 2022 16:30:10 -0700 Subject: bpf: Add bpf_skc_to_mptcp_sock_proto This patch implements a new struct bpf_func_proto, named bpf_skc_to_mptcp_sock_proto. Define a new bpf_id BTF_SOCK_TYPE_MPTCP, and a new helper bpf_skc_to_mptcp_sock(), which invokes another new helper bpf_mptcp_sock_from_subflow() in net/mptcp/bpf.c to get struct mptcp_sock from a given subflow socket. v2: Emit BTF type, add func_id checks in verifier.c and bpf_trace.c, remove build check for CONFIG_BPF_JIT v5: Drop EXPORT_SYMBOL (Martin) Co-developed-by: Nicolas Rybowski Co-developed-by: Matthieu Baerts Signed-off-by: Nicolas Rybowski Signed-off-by: Matthieu Baerts Signed-off-by: Geliang Tang Signed-off-by: Mat Martineau Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20220519233016.105670-2-mathew.j.martineau@linux.intel.com --- scripts/bpf_doc.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'scripts/bpf_doc.py') diff --git a/scripts/bpf_doc.py b/scripts/bpf_doc.py index 096625242475..d5452f7eb996 100755 --- a/scripts/bpf_doc.py +++ b/scripts/bpf_doc.py @@ -633,6 +633,7 @@ class PrinterHelpers(Printer): 'struct socket', 'struct file', 'struct bpf_timer', + 'struct mptcp_sock', ] known_types = { '...', @@ -682,6 +683,7 @@ class PrinterHelpers(Printer): 'struct socket', 'struct file', 'struct bpf_timer', + 'struct mptcp_sock', } mapped_types = { 'u8': '__u8', -- cgit From 97e03f521050c092919591e668107b3d69c5f426 Mon Sep 17 00:00:00 2001 From: Joanne Koong Date: Mon, 23 May 2022 14:07:07 -0700 Subject: bpf: Add verifier support for dynptrs This patch adds the bulk of the verifier work for supporting dynamic pointers (dynptrs) in bpf. A bpf_dynptr is opaque to the bpf program. It is a 16-byte structure defined internally as: struct bpf_dynptr_kern { void *data; u32 size; u32 offset; } __aligned(8); The upper 8 bits of *size* is reserved (it contains extra metadata about read-only status and dynptr type). Consequently, a dynptr only supports memory less than 16 MB. There are different types of dynptrs (eg malloc, ringbuf, ...). In this patchset, the most basic one, dynptrs to a bpf program's local memory, is added. For now only local memory that is of reg type PTR_TO_MAP_VALUE is supported. In the verifier, dynptr state information will be tracked in stack slots. When the program passes in an uninitialized dynptr (ARG_PTR_TO_DYNPTR | MEM_UNINIT), the stack slots corresponding to the frame pointer where the dynptr resides at are marked STACK_DYNPTR. For helper functions that take in initialized dynptrs (eg bpf_dynptr_read + bpf_dynptr_write which are added later in this patchset), the verifier enforces that the dynptr has been initialized properly by checking that their corresponding stack slots have been marked as STACK_DYNPTR. The 6th patch in this patchset adds test cases that the verifier should successfully reject, such as for example attempting to use a dynptr after doing a direct write into it inside the bpf program. Signed-off-by: Joanne Koong Signed-off-by: Andrii Nakryiko Acked-by: Andrii Nakryiko Acked-by: David Vernet Link: https://lore.kernel.org/bpf/20220523210712.3641569-2-joannelkoong@gmail.com --- scripts/bpf_doc.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'scripts/bpf_doc.py') diff --git a/scripts/bpf_doc.py b/scripts/bpf_doc.py index d5452f7eb996..855b937e7585 100755 --- a/scripts/bpf_doc.py +++ b/scripts/bpf_doc.py @@ -634,6 +634,7 @@ class PrinterHelpers(Printer): 'struct file', 'struct bpf_timer', 'struct mptcp_sock', + 'struct bpf_dynptr', ] known_types = { '...', @@ -684,6 +685,7 @@ class PrinterHelpers(Printer): 'struct file', 'struct bpf_timer', 'struct mptcp_sock', + 'struct bpf_dynptr', } mapped_types = { 'u8': '__u8', -- cgit From 33bf9885040c399cf6a95bd33216644126728e14 Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Wed, 15 Jun 2022 16:48:44 +0300 Subject: bpf: Add helpers to issue and check SYN cookies in XDP The new helpers bpf_tcp_raw_{gen,check}_syncookie_ipv{4,6} allow an XDP program to generate SYN cookies in response to TCP SYN packets and to check those cookies upon receiving the first ACK packet (the final packet of the TCP handshake). Unlike bpf_tcp_{gen,check}_syncookie these new helpers don't need a listening socket on the local machine, which allows to use them together with synproxy to accelerate SYN cookie generation. Signed-off-by: Maxim Mikityanskiy Reviewed-by: Tariq Toukan Link: https://lore.kernel.org/r/20220615134847.3753567-4-maximmi@nvidia.com Signed-off-by: Alexei Starovoitov --- scripts/bpf_doc.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'scripts/bpf_doc.py') diff --git a/scripts/bpf_doc.py b/scripts/bpf_doc.py index 855b937e7585..a0ec321469bd 100755 --- a/scripts/bpf_doc.py +++ b/scripts/bpf_doc.py @@ -635,6 +635,8 @@ class PrinterHelpers(Printer): 'struct bpf_timer', 'struct mptcp_sock', 'struct bpf_dynptr', + 'struct iphdr', + 'struct ipv6hdr', ] known_types = { '...', @@ -686,6 +688,8 @@ class PrinterHelpers(Printer): 'struct bpf_timer', 'struct mptcp_sock', 'struct bpf_dynptr', + 'struct iphdr', + 'struct ipv6hdr', } mapped_types = { 'u8': '__u8', -- cgit From 5cb62b7598f2413f54650d5124d7863e23f82a44 Mon Sep 17 00:00:00 2001 From: Alejandro Colomar Date: Thu, 21 Jul 2022 13:08:22 +0200 Subject: bpf, docs: Use SPDX license identifier in bpf_doc.py The Linux man-pages project now uses SPDX tags, instead of the full license text. Signed-off-by: Alejandro Colomar Signed-off-by: Daniel Borkmann Link: https://www.kernel.org/doc/man-pages/licenses.html Link: https://spdx.org/licenses/Linux-man-pages-copyleft.html Link: https://lore.kernel.org/bpf/20220721110821.8240-1-alx.manpages@gmail.com --- scripts/bpf_doc.py | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) (limited to 'scripts/bpf_doc.py') diff --git a/scripts/bpf_doc.py b/scripts/bpf_doc.py index a0ec321469bd..dfb260de17a8 100755 --- a/scripts/bpf_doc.py +++ b/scripts/bpf_doc.py @@ -333,27 +333,7 @@ class PrinterRST(Printer): .. Copyright (C) All BPF authors and contributors from 2014 to present. .. See git log include/uapi/linux/bpf.h in kernel tree for details. .. -.. %%%LICENSE_START(VERBATIM) -.. Permission is granted to make and distribute verbatim copies of this -.. manual provided the copyright notice and this permission notice are -.. preserved on all copies. -.. -.. Permission is granted to copy and distribute modified versions of this -.. manual under the conditions for verbatim copying, provided that the -.. entire resulting derived work is distributed under the terms of a -.. permission notice identical to this one. -.. -.. Since the Linux kernel and libraries are constantly changing, this -.. manual page may be incorrect or out-of-date. The author(s) assume no -.. responsibility for errors or omissions, or for damages resulting from -.. the use of the information contained herein. The author(s) may not -.. have taken the same level of care in the production of this manual, -.. which is licensed free of charge, as they might when working -.. professionally. -.. -.. Formatted or processed versions of this manual, if unaccompanied by -.. the source, must acknowledge the copyright and authors of this work. -.. %%%LICENSE_END +.. SPDX-License-Identifier: Linux-man-pages-copyleft .. .. Please do not edit this file. It was generated from the documentation .. located in file include/uapi/linux/bpf.h of the Linux kernel sources -- cgit From fd0a38f9c37d539f5603f887cdb637a4e6e6944d Mon Sep 17 00:00:00 2001 From: Quentin Monnet Date: Tue, 23 Aug 2022 16:53:26 +0100 Subject: scripts/bpf: Set version attribute for bpf-helpers(7) man page The bpf-helpers(7) manual page shipped in the man-pages project is generated from the documentation contained in the BPF UAPI header, in the Linux repository, parsed by script/bpf_doc.py and then fed to rst2man. After a recent update of that page [0], Alejandro reported that the linter used to validate the man pages complains about the generated document [1]. The header for the page is supposed to contain some attributes that we do not set correctly with the script. This commit updates the "project and version" field. We discussed the format of those fields in [1] and [2]. Before: $ ./scripts/bpf_doc.py helpers | rst2man | grep '\.TH' .TH BPF-HELPERS 7 "" "" "" After: $ ./scripts/bpf_doc.py helpers | rst2man | grep '\.TH' .TH BPF-HELPERS 7 "" "Linux v5.19-14022-g30d2a4d74e11" "" We get the version from "git describe", but if unavailable, we fall back on "make kernelversion". If none works, for example because neither git nore make are installed, we just set the field to "Linux" and keep generating the page. [0] https://git.kernel.org/pub/scm/docs/man-pages/man-pages.git/commit/man7/bpf-helpers.7?id=19c7f78393f2b038e76099f87335ddf43a87f039 [1] https://lore.kernel.org/all/20220823084719.13613-1-quentin@isovalent.com/t/#m58a418a318642c6428e14ce9bb84eba5183b06e8 [2] https://lore.kernel.org/all/20220721110821.8240-1-alx.manpages@gmail.com/t/#m8e689a822e03f6e2530a0d6de9d128401916c5de Reported-by: Alejandro Colomar Signed-off-by: Quentin Monnet Signed-off-by: Daniel Borkmann Reviewed-by: Alejandro Colomar Link: https://lore.kernel.org/bpf/20220823155327.98888-1-quentin@isovalent.com --- scripts/bpf_doc.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) (limited to 'scripts/bpf_doc.py') diff --git a/scripts/bpf_doc.py b/scripts/bpf_doc.py index dfb260de17a8..061ad1dc3212 100755 --- a/scripts/bpf_doc.py +++ b/scripts/bpf_doc.py @@ -10,6 +10,8 @@ from __future__ import print_function import argparse import re import sys, os +import subprocess + class NoHelperFound(BaseException): pass @@ -357,6 +359,20 @@ class PrinterRST(Printer): print('') + def get_kernel_version(self): + try: + version = subprocess.run(['git', 'describe'], cwd=linuxRoot, + capture_output=True, check=True) + version = version.stdout.decode().rstrip() + except: + try: + version = subprocess.run(['make', 'kernelversion'], cwd=linuxRoot, + capture_output=True, check=True) + version = version.stdout.decode().rstrip() + except: + return 'Linux' + return 'Linux {version}'.format(version=version) + class PrinterHelpersRST(PrinterRST): """ A printer for dumping collected information about helpers as a ReStructured @@ -378,6 +394,7 @@ list of eBPF helper functions ------------------------------------------------------------------------------- :Manual section: 7 +:Version: {version} DESCRIPTION =========== @@ -410,8 +427,10 @@ kernel at the top). HELPERS ======= ''' + kernelVersion = self.get_kernel_version() + PrinterRST.print_license(self) - print(header) + print(header.format(version=kernelVersion)) def print_footer(self): footer = ''' -- cgit From 92ec1cc3784a2a8a7a62596dcec4f2224b85dcf4 Mon Sep 17 00:00:00 2001 From: Quentin Monnet Date: Tue, 23 Aug 2022 16:53:27 +0100 Subject: scripts/bpf: Set date attribute for bpf-helpers(7) man page The bpf-helpers(7) manual page shipped in the man-pages project is generated from the documentation contained in the BPF UAPI header, in the Linux repository, parsed by script/bpf_doc.py and then fed to rst2man. The man page should contain the date of last modification of the documentation. This commit adds the relevant date when generating the page. Before: $ ./scripts/bpf_doc.py helpers | rst2man | grep '\.TH' .TH BPF-HELPERS 7 "" "Linux v5.19-14022-g30d2a4d74e11" "" After: $ ./scripts/bpf_doc.py helpers | rst2man | grep '\.TH' .TH BPF-HELPERS 7 "2022-08-15" "Linux v5.19-14022-g30d2a4d74e11" "" We get the version by using "git log" to look for the commit date of the latest change to the section of the BPF header containing the documentation. If the command fails, we just skip the date field. and keep generating the page. Reported-by: Alejandro Colomar Signed-off-by: Quentin Monnet Signed-off-by: Daniel Borkmann Reviewed-by: Alejandro Colomar Link: https://lore.kernel.org/bpf/20220823155327.98888-2-quentin@isovalent.com --- scripts/bpf_doc.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) (limited to 'scripts/bpf_doc.py') diff --git a/scripts/bpf_doc.py b/scripts/bpf_doc.py index 061ad1dc3212..f4f3e7ec6d44 100755 --- a/scripts/bpf_doc.py +++ b/scripts/bpf_doc.py @@ -12,6 +12,7 @@ import re import sys, os import subprocess +helpersDocStart = 'Start of BPF helper function descriptions:' class NoHelperFound(BaseException): pass @@ -235,7 +236,7 @@ class HeaderParser(object): self.enum_syscalls = re.findall('(BPF\w+)+', bpf_cmd_str) def parse_desc_helpers(self): - self.seek_to('* Start of BPF helper function descriptions:', + self.seek_to(helpersDocStart, 'Could not find start of eBPF helper descriptions list') while True: try: @@ -373,6 +374,17 @@ class PrinterRST(Printer): return 'Linux' return 'Linux {version}'.format(version=version) + def get_last_doc_update(self, delimiter): + try: + cmd = ['git', 'log', '-1', '--pretty=format:%cs', '--no-patch', + '-L', + '/{}/,/\*\//:include/uapi/linux/bpf.h'.format(delimiter)] + date = subprocess.run(cmd, cwd=linuxRoot, + capture_output=True, check=True) + return date.stdout.decode().rstrip() + except: + return '' + class PrinterHelpersRST(PrinterRST): """ A printer for dumping collected information about helpers as a ReStructured @@ -395,6 +407,7 @@ list of eBPF helper functions :Manual section: 7 :Version: {version} +{date_field}{date} DESCRIPTION =========== @@ -428,9 +441,12 @@ HELPERS ======= ''' kernelVersion = self.get_kernel_version() + lastUpdate = self.get_last_doc_update(helpersDocStart) PrinterRST.print_license(self) - print(header.format(version=kernelVersion)) + print(header.format(version=kernelVersion, + date_field = ':Date: ' if lastUpdate else '', + date=lastUpdate)) def print_footer(self): footer = ''' -- cgit From 0a0d55ef3e61d9f14e803cacb644fcc890f16774 Mon Sep 17 00:00:00 2001 From: Eyal Birger Date: Wed, 24 Aug 2022 21:10:43 +0300 Subject: bpf/scripts: Assert helper enum value is aligned with comment order The helper value is ABI as defined by enum bpf_func_id. As bpf_helper_defs.h is used for the userpace part, it must be consistent with this enum. Before this change the comments order was used by the bpf_doc script in order to set the helper values defined in the helpers file. When adding new helpers it is very puzzling when the userspace application breaks in weird places if the comment is inserted instead of appended - because the generated helper ABI is incorrect and shifted. This commit sets the helper value to the enum value. In addition it is currently the practice to have the comments appended and kept in the same order as the enum. As such, add an assertion validating the comment order is consistent with enum value. In case a different comments ordering is desired, this assertion can be lifted. Signed-off-by: Eyal Birger Signed-off-by: Andrii Nakryiko Reviewed-by: Quentin Monnet Link: https://lore.kernel.org/bpf/20220824181043.1601429-1-eyal.birger@gmail.com --- scripts/bpf_doc.py | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) (limited to 'scripts/bpf_doc.py') diff --git a/scripts/bpf_doc.py b/scripts/bpf_doc.py index f4f3e7ec6d44..d5c389df6045 100755 --- a/scripts/bpf_doc.py +++ b/scripts/bpf_doc.py @@ -50,6 +50,10 @@ class Helper(APIElement): @desc: textual description of the helper function @ret: description of the return value of the helper function """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.enum_val = None + def proto_break_down(self): """ Break down helper function protocol into smaller chunks: return type, @@ -92,6 +96,7 @@ class HeaderParser(object): self.commands = [] self.desc_unique_helpers = set() self.define_unique_helpers = [] + self.helper_enum_vals = {} self.desc_syscalls = [] self.enum_syscalls = [] @@ -248,30 +253,54 @@ class HeaderParser(object): break def parse_define_helpers(self): - # Parse the number of FN(...) in #define __BPF_FUNC_MAPPER to compare - # later with the number of unique function names present in description. + # Parse FN(...) in #define __BPF_FUNC_MAPPER to compare later with the + # number of unique function names present in description and use the + # correct enumeration value. # Note: seek_to(..) discards the first line below the target search text, # resulting in FN(unspec) being skipped and not added to self.define_unique_helpers. self.seek_to('#define __BPF_FUNC_MAPPER(FN)', 'Could not find start of eBPF helper definition list') - # Searches for either one or more FN(\w+) defines or a backslash for newline - p = re.compile('\s*(FN\(\w+\))+|\\\\') + # Searches for one FN(\w+) define or a backslash for newline + p = re.compile('\s*FN\((\w+)\)|\\\\') fn_defines_str = '' + i = 1 # 'unspec' is skipped as mentioned above while True: capture = p.match(self.line) if capture: fn_defines_str += self.line + self.helper_enum_vals[capture.expand(r'bpf_\1')] = i + i += 1 else: break self.line = self.reader.readline() # Find the number of occurences of FN(\w+) self.define_unique_helpers = re.findall('FN\(\w+\)', fn_defines_str) + def assign_helper_values(self): + seen_helpers = set() + for helper in self.helpers: + proto = helper.proto_break_down() + name = proto['name'] + try: + enum_val = self.helper_enum_vals[name] + except KeyError: + raise Exception("Helper %s is missing from enum bpf_func_id" % name) + + # Enforce current practice of having the descriptions ordered + # by enum value. + seen_helpers.add(name) + desc_val = len(seen_helpers) + if desc_val != enum_val: + raise Exception("Helper %s comment order (#%d) must be aligned with its position (#%d) in enum bpf_func_id" % (name, desc_val, enum_val)) + + helper.enum_val = enum_val + def run(self): self.parse_desc_syscall() self.parse_enum_syscall() self.parse_desc_helpers() self.parse_define_helpers() + self.assign_helper_values() self.reader.close() ############################################################################### @@ -796,7 +825,7 @@ class PrinterHelpers(Printer): comma = ', ' print(one_arg, end='') - print(') = (void *) %d;' % len(self.seen_helpers)) + print(') = (void *) %d;' % helper.enum_val) print('') ############################################################################### -- cgit