From ba99100687464b2b2f13a920c7d2cffbe8cac5c2 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Wed, 1 Oct 2025 16:49:24 +0200 Subject: tools: docs: parse_data_structs.py: drop contents header When used in practice, one may want to have multiple header files on a single rst file, like: *********************** Digital TV uAPI symbols *********************** .. contents:: Table of Contents :depth: 2 :local: Frontend ======== .. kernel-include:: include/uapi/linux/dvb/frontend.h :generate-cross-refs: :toc: Demux ===== .. kernel-include:: include/uapi/linux/dvb/dmx.h :generate-cross-refs: :toc: ... So, don't add ..contents:: here. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Message-ID: <4bf353e5248133a3b0abd82519a38453402fe7c6.1759329363.git.mchehab+huawei@kernel.org> --- tools/docs/lib/parse_data_structs.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'tools/docs/lib/parse_data_structs.py') diff --git a/tools/docs/lib/parse_data_structs.py b/tools/docs/lib/parse_data_structs.py index a5aa2e182052..6c868f84f029 100755 --- a/tools/docs/lib/parse_data_structs.py +++ b/tools/docs/lib/parse_data_structs.py @@ -397,16 +397,10 @@ class ParseDataStructs: def gen_toc(self): """ - Create a TOC table pointing to each symbol from the header + Create a list of symbols to be part of a TOC contents table """ text = [] - # Add header - text.append(".. contents:: Table of Contents") - text.append(" :depth: 2") - text.append(" :local:") - text.append("") - # Sort symbol types per description symbol_descriptions = [] for k, v in self.DEF_SYMBOL_TYPES.items(): -- cgit From ba9fbb3d9a4ba88c0b8713a7b5c86d58192fba22 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Wed, 1 Oct 2025 16:49:25 +0200 Subject: tools: docs: parse_data_structs.py: output a line number Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Message-ID: --- tools/docs/lib/parse_data_structs.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) (limited to 'tools/docs/lib/parse_data_structs.py') diff --git a/tools/docs/lib/parse_data_structs.py b/tools/docs/lib/parse_data_structs.py index 6c868f84f029..d28471a045f1 100755 --- a/tools/docs/lib/parse_data_structs.py +++ b/tools/docs/lib/parse_data_structs.py @@ -143,7 +143,7 @@ class ParseDataStructs: for symbol_type in self.DEF_SYMBOL_TYPES: self.symbols[symbol_type] = {} - def store_type(self, symbol_type: str, symbol: str, + def store_type(self, ln, symbol_type: str, symbol: str, ref_name: str = None, replace_underscores: bool = True): """ Stores a new symbol at self.symbols under symbol_type. @@ -172,7 +172,7 @@ class ParseDataStructs: else: ref_link = symbol - self.symbols[symbol_type][symbol] = f"{prefix}{ref_link}{suffix}" + self.symbols[symbol_type][symbol] = (f"{prefix}{ref_link}{suffix}", ln) def store_line(self, line): """Stores a line at self.data, properly indented""" @@ -240,20 +240,20 @@ class ParseDataStructs: if is_enum: match = re.match(r"^\s*([_\w][\w\d_]+)\s*[\,=]?", line) if match: - self.store_type("symbol", match.group(1)) + self.store_type(line_no, "symbol", match.group(1)) if "}" in line: is_enum = False continue match = re.match(r"^\s*#\s*define\s+([\w_]+)\s+_IO", line) if match: - self.store_type("ioctl", match.group(1), + self.store_type(line_no, "ioctl", match.group(1), replace_underscores=False) continue match = re.match(r"^\s*#\s*define\s+([\w_]+)(\s+|$)", line) if match: - self.store_type("define", match.group(1)) + self.store_type(line_no, "define", match.group(1)) continue match = re.match(r"^\s*typedef\s+([_\w][\w\d_]+)\s+(.*)\s+([_\w][\w\d_]+);", @@ -261,20 +261,20 @@ class ParseDataStructs: if match: name = match.group(2).strip() symbol = match.group(3) - self.store_type("typedef", symbol, ref_name=name) + self.store_type(line_no, "typedef", symbol, ref_name=name) continue for re_enum in self.RE_ENUMS: match = re_enum.match(line) if match: - self.store_type("enum", match.group(1)) + self.store_type(line_no, "enum", match.group(1)) is_enum = True break for re_struct in self.RE_STRUCTS: match = re_struct.match(line) if match: - self.store_type("struct", match.group(1)) + self.store_type(line_no, "struct", match.group(1)) break def process_exceptions(self, fname: str): @@ -342,7 +342,8 @@ class ParseDataStructs: # Change self.symbols to use the replacement rule if old in self.symbols[c_type]: - self.symbols[c_type][old] = new_ref + (_, ln) = self.symbols[c_type][old] + self.symbols[c_type][old] = (new_ref, ln) else: print(f"{name}:{ln}: Warning: can't find {old} {c_type}") @@ -360,8 +361,8 @@ class ParseDataStructs: print(f"{c_type}:") - for symbol, ref in sorted(refs.items()): - print(f" {symbol} -> {ref}") + for symbol, (ref, ln) in sorted(refs.items()): + print(f" #{ln:<5d} {symbol} -> {ref}") print() @@ -384,7 +385,7 @@ class ParseDataStructs: # Process all reference types for ref_dict in self.symbols.values(): - for symbol, replacement in ref_dict.items(): + for symbol, (replacement, _) in ref_dict.items(): symbol = re.escape(re.sub(r"([\_\`\*\<\>\&\\\\:\/])", r"\\\1", symbol)) text = re.sub(fr'{start_delim}{symbol}{end_delim}', fr'\1{replacement}\2', text) @@ -420,8 +421,8 @@ class ParseDataStructs: text.append("") # Sort symbols alphabetically - for symbol, ref in sorted(refs.items()): - text.append(f"* :{ref}:") + for symbol, (ref, ln) in sorted(refs.items()): + text.append(f"* {ref}: line #{ln}") text.append("") # Add empty line between categories -- cgit From 3ed9521772880099803619b57056c8d3cec16f27 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Wed, 1 Oct 2025 16:49:26 +0200 Subject: docs: kernel_include.py: fix line numbers for TOC On TOC output, we need to embeed line numbers with ViewList. Change the parse class to produce a line-number parsed result, and adjust the output accordingly. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Message-ID: <74eed96e32f79eaaef7a99ffe7c3224fed369c27.1759329363.git.mchehab+huawei@kernel.org> --- tools/docs/lib/parse_data_structs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools/docs/lib/parse_data_structs.py') diff --git a/tools/docs/lib/parse_data_structs.py b/tools/docs/lib/parse_data_structs.py index d28471a045f1..9ad621712103 100755 --- a/tools/docs/lib/parse_data_structs.py +++ b/tools/docs/lib/parse_data_structs.py @@ -422,7 +422,7 @@ class ParseDataStructs: # Sort symbols alphabetically for symbol, (ref, ln) in sorted(refs.items()): - text.append(f"* {ref}: line #{ln}") + text.append(f"- LINENO_{ln}: {ref}") text.append("") # Add empty line between categories -- cgit From 9e4173432eecf7af9e6d8bf38fca537b55ed8538 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Wed, 1 Oct 2025 16:49:28 +0200 Subject: tools: docs: parse_data_structs: make process_exceptions two stages Split the logic which parses exceptions on two stages, preparing the exceptions file to have rules that will affect xref generation. For now, preserve the original API. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Message-ID: --- tools/docs/lib/parse_data_structs.py | 129 ++++++++++++++++++++--------------- 1 file changed, 73 insertions(+), 56 deletions(-) (limited to 'tools/docs/lib/parse_data_structs.py') diff --git a/tools/docs/lib/parse_data_structs.py b/tools/docs/lib/parse_data_structs.py index 9ad621712103..46535a05ea4a 100755 --- a/tools/docs/lib/parse_data_structs.py +++ b/tools/docs/lib/parse_data_structs.py @@ -140,9 +140,41 @@ class ParseDataStructs: self.symbols = {} + self.ignore = [] + self.replace = [] + for symbol_type in self.DEF_SYMBOL_TYPES: self.symbols[symbol_type] = {} + def read_exceptions(self, fname: str): + if not fname: + return + + name = os.path.basename(fname) + + with open(fname, "r", encoding="utf-8", errors="backslashreplace") as f: + for ln, line in enumerate(f): + ln += 1 + line = line.strip() + if not line or line.startswith("#"): + continue + + # ignore rules + match = re.match(r"^ignore\s+(\w+)\s+(\S+)", line) + + if match: + self.ignore.append((ln, match.group(1), match.group(2))) + continue + + # replace rules + match = re.match(r"^replace\s+(\S+)\s+(\S+)\s+(\S+)", line) + if match: + self.replace.append((ln, match.group(1), match.group(2), + match.group(3))) + continue + + sys.exit(f"{name}:{ln}: invalid line: {line}") + def store_type(self, ln, symbol_type: str, symbol: str, ref_name: str = None, replace_underscores: bool = True): """ @@ -277,75 +309,60 @@ class ParseDataStructs: self.store_type(line_no, "struct", match.group(1)) break - def process_exceptions(self, fname: str): + def apply_exceptions(self): """ Process exceptions file with rules to ignore or replace references. """ - if not fname: - return - name = os.path.basename(fname) + # Handle ignore rules + for ln, c_type, symbol in self.ignore: + if c_type not in self.DEF_SYMBOL_TYPES: + sys.exit(f"{name}:{ln}: {c_type} is invalid") - with open(fname, "r", encoding="utf-8", errors="backslashreplace") as f: - for ln, line in enumerate(f): - ln += 1 - line = line.strip() - if not line or line.startswith("#"): - continue - - # Handle ignore rules - match = re.match(r"^ignore\s+(\w+)\s+(\S+)", line) - if match: - c_type = match.group(1) - symbol = match.group(2) - - if c_type not in self.DEF_SYMBOL_TYPES: - sys.exit(f"{name}:{ln}: {c_type} is invalid") - - d = self.symbols[c_type] - if symbol in d: - del d[symbol] - - continue + d = self.symbols[c_type] + if symbol in d: + del d[symbol] - # Handle replace rules - match = re.match(r"^replace\s+(\S+)\s+(\S+)\s+(\S+)", line) - if not match: - sys.exit(f"{name}:{ln}: invalid line: {line}") - - c_type, old, new = match.groups() + # Handle replace rules + for ln, c_type, old, new in self.replace: + if c_type not in self.DEF_SYMBOL_TYPES: + sys.exit(f"{name}:{ln}: {c_type} is invalid") - if c_type not in self.DEF_SYMBOL_TYPES: - sys.exit(f"{name}:{ln}: {c_type} is invalid") + reftype = None - reftype = None + # Parse reference type when the type is specified - # Parse reference type when the type is specified - - match = re.match(r"^\:c\:(data|func|macro|type)\:\`(.+)\`", new) + match = re.match(r"^\:c\:(data|func|macro|type)\:\`(.+)\`", new) + if match: + reftype = f":c:{match.group(1)}" + new = match.group(2) + else: + match = re.search(r"(\:ref)\:\`(.+)\`", new) if match: - reftype = f":c:{match.group(1)}" + reftype = match.group(1) new = match.group(2) - else: - match = re.search(r"(\:ref)\:\`(.+)\`", new) - if match: - reftype = match.group(1) - new = match.group(2) - # If the replacement rule doesn't have a type, get default + # If the replacement rule doesn't have a type, get default + if not reftype: + reftype = self.DEF_SYMBOL_TYPES[c_type].get("ref_type") if not reftype: - reftype = self.DEF_SYMBOL_TYPES[c_type].get("ref_type") - if not reftype: - reftype = self.DEF_SYMBOL_TYPES[c_type].get("real_type") - - new_ref = f"{reftype}:`{old} <{new}>`" - - # Change self.symbols to use the replacement rule - if old in self.symbols[c_type]: - (_, ln) = self.symbols[c_type][old] - self.symbols[c_type][old] = (new_ref, ln) - else: - print(f"{name}:{ln}: Warning: can't find {old} {c_type}") + reftype = self.DEF_SYMBOL_TYPES[c_type].get("real_type") + + new_ref = f"{reftype}:`{old} <{new}>`" + + # Change self.symbols to use the replacement rule + if old in self.symbols[c_type]: + (_, ln) = self.symbols[c_type][old] + self.symbols[c_type][old] = (new_ref, ln) + else: + print(f"{name}:{ln}: Warning: can't find {old} {c_type}") + + def process_exceptions(self, fname: str): + """ + Process exceptions file with rules to ignore or replace references. + """ + self.read_exceptions(fname) + self.apply_exceptions() def debug_print(self): """ -- cgit From 2cdd27a70887f8206809fae5c2c08c768fd8daba Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Wed, 1 Oct 2025 16:49:29 +0200 Subject: tools: docs: parse_data_structs.py: get rid of process_exceptions() Add an extra parameter to parse_file to make it handle exceptions internally, cleaning up the API. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Message-ID: <8575bbc94ff706aa7e7cc3a188399ca17a3169e6.1759329363.git.mchehab+huawei@kernel.org> --- tools/docs/lib/parse_data_structs.py | 105 +++++++++++++++++------------------ 1 file changed, 51 insertions(+), 54 deletions(-) (limited to 'tools/docs/lib/parse_data_structs.py') diff --git a/tools/docs/lib/parse_data_structs.py b/tools/docs/lib/parse_data_structs.py index 46535a05ea4a..cbdbf7dfe785 100755 --- a/tools/docs/lib/parse_data_structs.py +++ b/tools/docs/lib/parse_data_structs.py @@ -175,6 +175,54 @@ class ParseDataStructs: sys.exit(f"{name}:{ln}: invalid line: {line}") + def apply_exceptions(self): + """ + Process exceptions file with rules to ignore or replace references. + """ + + # Handle ignore rules + for ln, c_type, symbol in self.ignore: + if c_type not in self.DEF_SYMBOL_TYPES: + sys.exit(f"{name}:{ln}: {c_type} is invalid") + + d = self.symbols[c_type] + if symbol in d: + del d[symbol] + + # Handle replace rules + for ln, c_type, old, new in self.replace: + if c_type not in self.DEF_SYMBOL_TYPES: + sys.exit(f"{name}:{ln}: {c_type} is invalid") + + reftype = None + + # Parse reference type when the type is specified + + match = re.match(r"^\:c\:(data|func|macro|type)\:\`(.+)\`", new) + if match: + reftype = f":c:{match.group(1)}" + new = match.group(2) + else: + match = re.search(r"(\:ref)\:\`(.+)\`", new) + if match: + reftype = match.group(1) + new = match.group(2) + + # If the replacement rule doesn't have a type, get default + if not reftype: + reftype = self.DEF_SYMBOL_TYPES[c_type].get("ref_type") + if not reftype: + reftype = self.DEF_SYMBOL_TYPES[c_type].get("real_type") + + new_ref = f"{reftype}:`{old} <{new}>`" + + # Change self.symbols to use the replacement rule + if old in self.symbols[c_type]: + (_, ln) = self.symbols[c_type][old] + self.symbols[c_type][old] = (new_ref, ln) + else: + print(f"{name}:{ln}: Warning: can't find {old} {c_type}") + def store_type(self, ln, symbol_type: str, symbol: str, ref_name: str = None, replace_underscores: bool = True): """ @@ -211,13 +259,15 @@ class ParseDataStructs: line = " " + line.expandtabs() self.data += line.rstrip(" ") - def parse_file(self, file_in: str): + def parse_file(self, file_in: str, exceptions: str = None): """Reads a C source file and get identifiers""" self.data = "" is_enum = False is_comment = False multiline = "" + self.read_exceptions(exceptions) + with open(file_in, "r", encoding="utf-8", errors="backslashreplace") as f: for line_no, line in enumerate(f): @@ -309,59 +359,6 @@ class ParseDataStructs: self.store_type(line_no, "struct", match.group(1)) break - def apply_exceptions(self): - """ - Process exceptions file with rules to ignore or replace references. - """ - - # Handle ignore rules - for ln, c_type, symbol in self.ignore: - if c_type not in self.DEF_SYMBOL_TYPES: - sys.exit(f"{name}:{ln}: {c_type} is invalid") - - d = self.symbols[c_type] - if symbol in d: - del d[symbol] - - # Handle replace rules - for ln, c_type, old, new in self.replace: - if c_type not in self.DEF_SYMBOL_TYPES: - sys.exit(f"{name}:{ln}: {c_type} is invalid") - - reftype = None - - # Parse reference type when the type is specified - - match = re.match(r"^\:c\:(data|func|macro|type)\:\`(.+)\`", new) - if match: - reftype = f":c:{match.group(1)}" - new = match.group(2) - else: - match = re.search(r"(\:ref)\:\`(.+)\`", new) - if match: - reftype = match.group(1) - new = match.group(2) - - # If the replacement rule doesn't have a type, get default - if not reftype: - reftype = self.DEF_SYMBOL_TYPES[c_type].get("ref_type") - if not reftype: - reftype = self.DEF_SYMBOL_TYPES[c_type].get("real_type") - - new_ref = f"{reftype}:`{old} <{new}>`" - - # Change self.symbols to use the replacement rule - if old in self.symbols[c_type]: - (_, ln) = self.symbols[c_type][old] - self.symbols[c_type][old] = (new_ref, ln) - else: - print(f"{name}:{ln}: Warning: can't find {old} {c_type}") - - def process_exceptions(self, fname: str): - """ - Process exceptions file with rules to ignore or replace references. - """ - self.read_exceptions(fname) self.apply_exceptions() def debug_print(self): -- cgit From 7f809e6a6f07e87621e8a6d9cc553fa763cc08af Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Wed, 1 Oct 2025 16:49:30 +0200 Subject: tools: docs: parse_data_structs.py: add namespace support C domain supports a ".. c:namespace::" tag that allows setting a symbol namespace. This is used within the kernel to avoid warnings about duplicated symbols. This is specially important for syscalls, as each subsystem may have their own documentation for them. This is specially true for ioctl. When such tag is used, all C domain symbols have c++ style, e.g. they'll become "{namespace}.{reference}". Allow specifying C namespace at the exception files, avoiding the need of override rules for every symbol. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Message-ID: --- tools/docs/lib/parse_data_structs.py | 43 +++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 11 deletions(-) (limited to 'tools/docs/lib/parse_data_structs.py') diff --git a/tools/docs/lib/parse_data_structs.py b/tools/docs/lib/parse_data_structs.py index cbdbf7dfe785..0d537e989ea7 100755 --- a/tools/docs/lib/parse_data_structs.py +++ b/tools/docs/lib/parse_data_structs.py @@ -53,11 +53,19 @@ class ParseDataStructs: replace - Replaces how old_symbol with a new reference. The new_reference can be: + Replaces how old_symbol with a new reference. The new_reference can be: + - A simple symbol name; - A full Sphinx reference. - On both cases, can be: + 3. Namespace rules + + namespace + + Sets C namespace to be used during cross-reference generation. Can + be overridden by replace rules. + + On ignore and replace rules, can be: - ioctl: for defines that end with _IO*, e.g. ioctl definitions - define: for other defines - symbol: for symbols defined within enums; @@ -71,6 +79,8 @@ class ParseDataStructs: ignore ioctl VIDIOC_ENUM_FMT replace ioctl VIDIOC_DQBUF vidioc_qbuf replace define V4L2_EVENT_MD_FL_HAVE_FRAME_SEQ :c:type:`v4l2_event_motion_det` + + namespace MC """ # Parser regexes with multiple ways to capture enums and structs @@ -140,6 +150,7 @@ class ParseDataStructs: self.symbols = {} + self.namespace = None self.ignore = [] self.replace = [] @@ -173,6 +184,11 @@ class ParseDataStructs: match.group(3))) continue + match = re.match(r"^namespace\s+(\S+)", line) + if match: + self.namespace = match.group(1) + continue + sys.exit(f"{name}:{ln}: invalid line: {line}") def apply_exceptions(self): @@ -237,18 +253,23 @@ class ParseDataStructs: ref_type = defs.get("ref_type") # Determine ref_link based on symbol type - if ref_type: - if symbol_type == "enum": - ref_link = f"{ref_type}:`{symbol}`" - else: - if not ref_name: - ref_name = symbol.lower() + if ref_type or self.namespace: + if not ref_name: + ref_name = symbol.lower() - # c-type references don't support hash - if ref_type == ":ref" and replace_underscores: - ref_name = ref_name.replace("_", "-") + # c-type references don't support hash + if ref_type == ":ref" and replace_underscores: + ref_name = ref_name.replace("_", "-") + # C domain references may have namespaces + if ref_type.startswith(":c:"): + if self.namespace: + ref_name = f"{self.namespace}.{ref_name}" + + if ref_type: ref_link = f"{ref_type}:`{symbol} <{ref_name}>`" + else: + ref_link = f"`{symbol} <{ref_name}>`" else: ref_link = symbol -- cgit From d2a72e1f27c16d23938b26a78d7e1644b06bb5aa Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Wed, 1 Oct 2025 16:49:31 +0200 Subject: tools: docs: parse_data_structs.py: accept more reftypes Current regex is limited to only some c-domain reftypes. There are several others. Change the code to pick the name specified there. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Message-ID: <5c146923d1e3183893f290216fb1378954e2e540.1759329363.git.mchehab+huawei@kernel.org> --- tools/docs/lib/parse_data_structs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools/docs/lib/parse_data_structs.py') diff --git a/tools/docs/lib/parse_data_structs.py b/tools/docs/lib/parse_data_structs.py index 0d537e989ea7..25361996cd20 100755 --- a/tools/docs/lib/parse_data_structs.py +++ b/tools/docs/lib/parse_data_structs.py @@ -214,7 +214,7 @@ class ParseDataStructs: # Parse reference type when the type is specified - match = re.match(r"^\:c\:(data|func|macro|type)\:\`(.+)\`", new) + match = re.match(r"^\:c\:(\w+)\:\`(.+)\`", new) if match: reftype = f":c:{match.group(1)}" new = match.group(2) -- cgit