From ee13b3f35c7238eff323450599185a775d726462 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:12 +0800 Subject: scripts/kernel-doc.py: move KernelFiles class to a separate file The KernelFiles class is the main dispatcher which parses each source file. In preparation for letting kerneldoc Sphinx extension to import Python libraries, move regex ancillary classes to a separate file. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/80bc855e128a9ff0a11df5afe9ba71775dfc9a0f.1744106241.git.mchehab+huawei@kernel.org --- scripts/lib/kdoc/kdoc_files.py | 270 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100755 scripts/lib/kdoc/kdoc_files.py (limited to 'scripts/lib/kdoc/kdoc_files.py') diff --git a/scripts/lib/kdoc/kdoc_files.py b/scripts/lib/kdoc/kdoc_files.py new file mode 100755 index 000000000000..8bcdc7ead984 --- /dev/null +++ b/scripts/lib/kdoc/kdoc_files.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2025: Mauro Carvalho Chehab . +# +# pylint: disable=R0903,R0913,R0914,R0917 + +# TODO: implement warning filtering + +""" +Parse lernel-doc tags on multiple kernel source files. +""" + +import argparse +import logging +import os +import re +import sys +from datetime import datetime + +from dateutil import tz + +from kdoc_parser import KernelDoc + + +class GlobSourceFiles: + """ + Parse C source code file names and directories via an Interactor. + """ + + def __init__(self, srctree=None, valid_extensions=None): + """ + Initialize valid extensions with a tuple. + + If not defined, assume default C extensions (.c and .h) + + It would be possible to use python's glob function, but it is + very slow, and it is not interactive. So, it would wait to read all + directories before actually do something. + + So, let's use our own implementation. + """ + + if not valid_extensions: + self.extensions = (".c", ".h") + else: + self.extensions = valid_extensions + + self.srctree = srctree + + def _parse_dir(self, dirname): + """Internal function to parse files recursively""" + + with os.scandir(dirname) as obj: + for entry in obj: + name = os.path.join(dirname, entry.name) + + if entry.is_dir(): + yield from self._parse_dir(name) + + if not entry.is_file(): + continue + + basename = os.path.basename(name) + + if not basename.endswith(self.extensions): + continue + + yield name + + def parse_files(self, file_list, file_not_found_cb): + """ + Define an interator to parse all source files from file_list, + handling directories if any + """ + + for fname in file_list: + if self.srctree: + f = os.path.join(self.srctree, fname) + else: + f = fname + + if os.path.isdir(f): + yield from self._parse_dir(f) + elif os.path.isfile(f): + yield f + elif file_not_found_cb: + file_not_found_cb(fname) + + +class KernelFiles(): + """ + Parse lernel-doc tags on multiple kernel source files. + """ + + def parse_file(self, fname): + """ + Parse a single Kernel source. + """ + + doc = KernelDoc(self.config, fname) + doc.run() + + return doc + + def process_export_file(self, fname): + """ + Parses EXPORT_SYMBOL* macros from a single Kernel source file. + """ + try: + with open(fname, "r", encoding="utf8", + errors="backslashreplace") as fp: + for line in fp: + KernelDoc.process_export(self.config.function_table, line) + + except IOError: + print(f"Error: Cannot open fname {fname}", fname=sys.stderr) + self.config.errors += 1 + + def file_not_found_cb(self, fname): + """ + Callback to warn if a file was not found. + """ + + self.config.log.error("Cannot find file %s", fname) + self.config.errors += 1 + + def __init__(self, files=None, verbose=False, out_style=None, + werror=False, wreturn=False, wshort_desc=False, + wcontents_before_sections=False, + logger=None, modulename=None, export_file=None): + """ + Initialize startup variables and parse all files + """ + + if not verbose: + verbose = bool(os.environ.get("KBUILD_VERBOSE", 0)) + + if not modulename: + modulename = "Kernel API" + + dt = datetime.now() + if os.environ.get("KBUILD_BUILD_TIMESTAMP", None): + # use UTC TZ + to_zone = tz.gettz('UTC') + dt = dt.astimezone(to_zone) + + if not werror: + kcflags = os.environ.get("KCFLAGS", None) + if kcflags: + match = re.search(r"(\s|^)-Werror(\s|$)/", kcflags) + if match: + werror = True + + # reading this variable is for backwards compat just in case + # someone was calling it with the variable from outside the + # kernel's build system + kdoc_werror = os.environ.get("KDOC_WERROR", None) + if kdoc_werror: + werror = kdoc_werror + + # Set global config data used on all files + self.config = argparse.Namespace + + self.config.verbose = verbose + self.config.werror = werror + self.config.wreturn = wreturn + self.config.wshort_desc = wshort_desc + self.config.wcontents_before_sections = wcontents_before_sections + self.config.modulename = modulename + + self.config.function_table = set() + self.config.source_map = {} + + if not logger: + self.config.log = logging.getLogger("kernel-doc") + else: + self.config.log = logger + + self.config.kernel_version = os.environ.get("KERNELVERSION", + "unknown kernel version'") + self.config.src_tree = os.environ.get("SRCTREE", None) + + self.out_style = out_style + self.export_file = export_file + + # Initialize internal variables + + self.config.errors = 0 + self.results = [] + + self.file_list = files + self.files = set() + + def parse(self): + """ + Parse all files + """ + + glob = GlobSourceFiles(srctree=self.config.src_tree) + + # Let's use a set here to avoid duplicating files + + for fname in glob.parse_files(self.file_list, self.file_not_found_cb): + if fname in self.files: + continue + + self.files.add(fname) + + res = self.parse_file(fname) + self.results.append((res.fname, res.entries)) + + if not self.files: + sys.exit(1) + + # If a list of export files was provided, parse EXPORT_SYMBOL* + # from the ones not already parsed + + if self.export_file: + files = self.files + + glob = GlobSourceFiles(srctree=self.config.src_tree) + + for fname in glob.parse_files(self.export_file, + self.file_not_found_cb): + if fname not in files: + files.add(fname) + + self.process_export_file(fname) + + def out_msg(self, fname, name, arg): + """ + Output messages from a file name using the output style filtering. + + If output type was not handled by the syler, return False. + """ + + # NOTE: we can add rules here to filter out unwanted parts, + # although OutputFormat.msg already does that. + + return self.out_style.msg(fname, name, arg) + + def msg(self, enable_lineno=False, export=False, internal=False, + symbol=None, nosymbol=None): + """ + Interacts over the kernel-doc results and output messages. + """ + + function_table = self.config.function_table + + if symbol: + for s in symbol: + function_table.add(s) + + # Output none mode: only warnings will be shown + if not self.out_style: + return + + self.out_style.set_config(self.config) + + self.out_style.set_filter(export, internal, symbol, nosymbol, + function_table, enable_lineno) + + for fname, arg_tuple in self.results: + for name, arg in arg_tuple: + if self.out_msg(fname, name, arg): + ln = arg.get("ln", 0) + dtype = arg.get('type', "") + + self.config.log.warning("%s:%d Can't handle %s", + fname, ln, dtype) -- cgit From 4fa5e411379af1baabdff088196da977799fd46e Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:14 +0800 Subject: scripts/kernel-doc.py: convert message output to an interactor Instead of directly printing output messages, change kdoc classes to return an interactor with the output message, letting the actual display to happen at the command-line command. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/557304c8458f1fb4aa2e833f4bdaff953094ddcb.1744106242.git.mchehab+huawei@kernel.org --- scripts/lib/kdoc/kdoc_files.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'scripts/lib/kdoc/kdoc_files.py') diff --git a/scripts/lib/kdoc/kdoc_files.py b/scripts/lib/kdoc/kdoc_files.py index 8bcdc7ead984..817ed98b2727 100755 --- a/scripts/lib/kdoc/kdoc_files.py +++ b/scripts/lib/kdoc/kdoc_files.py @@ -229,9 +229,10 @@ class KernelFiles(): def out_msg(self, fname, name, arg): """ - Output messages from a file name using the output style filtering. + Return output messages from a file name using the output style + filtering. - If output type was not handled by the syler, return False. + If output type was not handled by the syler, return None. """ # NOTE: we can add rules here to filter out unwanted parts, @@ -242,7 +243,8 @@ class KernelFiles(): def msg(self, enable_lineno=False, export=False, internal=False, symbol=None, nosymbol=None): """ - Interacts over the kernel-doc results and output messages. + Interacts over the kernel-doc results and output messages, + returning kernel-doc markups on each interaction """ function_table = self.config.function_table @@ -261,10 +263,15 @@ class KernelFiles(): function_table, enable_lineno) for fname, arg_tuple in self.results: + msg = "" for name, arg in arg_tuple: - if self.out_msg(fname, name, arg): + msg += self.out_msg(fname, name, arg) + + if msg is None: ln = arg.get("ln", 0) dtype = arg.get('type', "") self.config.log.warning("%s:%d Can't handle %s", fname, ln, dtype) + if msg: + yield fname, msg -- cgit From 799b0d2a2a24f09f3800678f55ad48d8cc9c069d Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:15 +0800 Subject: scripts/kernel-doc.py: move file lists to the parser function Instead of setting file lists at __init__ time, move it to the actual parsing function. This allows adding more files to be parsed in real time, by calling parse function multiple times. With the new way, the export_files logic was rewritten to avoid parsing twice EXPORT_SYMBOL for partial matches. Please notice that, with this logic, it can still read the same file twice when export_file is used. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/ab10bc94050406ce6536d4944b5d718ecd70812f.1744106242.git.mchehab+huawei@kernel.org --- scripts/lib/kdoc/kdoc_files.py | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) (limited to 'scripts/lib/kdoc/kdoc_files.py') diff --git a/scripts/lib/kdoc/kdoc_files.py b/scripts/lib/kdoc/kdoc_files.py index 817ed98b2727..47dab46c89fe 100755 --- a/scripts/lib/kdoc/kdoc_files.py +++ b/scripts/lib/kdoc/kdoc_files.py @@ -124,7 +124,7 @@ class KernelFiles(): self.config.log.error("Cannot find file %s", fname) self.config.errors += 1 - def __init__(self, files=None, verbose=False, out_style=None, + def __init__(self, verbose=False, out_style=None, werror=False, wreturn=False, wshort_desc=False, wcontents_before_sections=False, logger=None, modulename=None, export_file=None): @@ -181,51 +181,48 @@ class KernelFiles(): self.config.src_tree = os.environ.get("SRCTREE", None) self.out_style = out_style - self.export_file = export_file # Initialize internal variables self.config.errors = 0 self.results = [] - self.file_list = files self.files = set() + self.export_files = set() - def parse(self): + def parse(self, file_list, export_file=None): """ Parse all files """ glob = GlobSourceFiles(srctree=self.config.src_tree) - # Let's use a set here to avoid duplicating files + # Prevent parsing the same file twice to speedup parsing and + # avoid reporting errors multiple times - for fname in glob.parse_files(self.file_list, self.file_not_found_cb): + for fname in glob.parse_files(file_list, self.file_not_found_cb): if fname in self.files: continue - self.files.add(fname) - res = self.parse_file(fname) - self.results.append((res.fname, res.entries)) - if not self.files: - sys.exit(1) + self.results.append((res.fname, res.entries)) + self.files.add(fname) # If a list of export files was provided, parse EXPORT_SYMBOL* - # from the ones not already parsed + # from files that weren't fully parsed - if self.export_file: - files = self.files + if not export_file: + return - glob = GlobSourceFiles(srctree=self.config.src_tree) + self.export_files |= self.files - for fname in glob.parse_files(self.export_file, - self.file_not_found_cb): - if fname not in files: - files.add(fname) + glob = GlobSourceFiles(srctree=self.config.src_tree) - self.process_export_file(fname) + for fname in glob.parse_files(export_file, self.file_not_found_cb): + if fname not in self.export_files: + self.process_export_file(fname) + self.export_files.add(fname) def out_msg(self, fname, name, arg): """ -- cgit From 0873e55433769210c0ba26227f0080dde408e15e Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:16 +0800 Subject: scripts/kernel-doc.py: implement support for -no-doc-sections The venerable kernel-doc Perl script has a number of options that aren't properly documented. Among them, there is -no-doc-sections, which is used by the Sphinx extension. Implement support for it. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/06b18a32142b44d5ba8b41ac64a76c02b03b4969.1744106242.git.mchehab+huawei@kernel.org --- scripts/lib/kdoc/kdoc_files.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'scripts/lib/kdoc/kdoc_files.py') diff --git a/scripts/lib/kdoc/kdoc_files.py b/scripts/lib/kdoc/kdoc_files.py index 47dab46c89fe..4c04546a74fe 100755 --- a/scripts/lib/kdoc/kdoc_files.py +++ b/scripts/lib/kdoc/kdoc_files.py @@ -238,7 +238,7 @@ class KernelFiles(): return self.out_style.msg(fname, name, arg) def msg(self, enable_lineno=False, export=False, internal=False, - symbol=None, nosymbol=None): + symbol=None, nosymbol=None, no_doc_sections=False): """ Interacts over the kernel-doc results and output messages, returning kernel-doc markups on each interaction @@ -257,7 +257,8 @@ class KernelFiles(): self.out_style.set_config(self.config) self.out_style.set_filter(export, internal, symbol, nosymbol, - function_table, enable_lineno) + function_table, enable_lineno, + no_doc_sections) for fname, arg_tuple in self.results: msg = "" -- cgit From 43ecfe6bc2ae11f99fb4b812c014339c2d6a221a Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:25 +0800 Subject: scripts/kernel-doc.py: Set an output format for --none Now that warnings output is deferred to the output plugin, we need to have an output style for none as well. So, use the OutputFormat base class on such cases. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/caa1089e16f4609f792ff26731ad9e9c3a6f6b1d.1744106242.git.mchehab+huawei@kernel.org --- scripts/lib/kdoc/kdoc_files.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'scripts/lib/kdoc/kdoc_files.py') diff --git a/scripts/lib/kdoc/kdoc_files.py b/scripts/lib/kdoc/kdoc_files.py index 4c04546a74fe..dd3dbe87520b 100755 --- a/scripts/lib/kdoc/kdoc_files.py +++ b/scripts/lib/kdoc/kdoc_files.py @@ -20,6 +20,7 @@ from datetime import datetime from dateutil import tz from kdoc_parser import KernelDoc +from kdoc_output import OutputFormat class GlobSourceFiles: @@ -138,6 +139,9 @@ class KernelFiles(): if not modulename: modulename = "Kernel API" + if out_style is None: + out_style = OutputFormat() + dt = datetime.now() if os.environ.get("KBUILD_BUILD_TIMESTAMP", None): # use UTC TZ -- cgit From 485f6f7960c468d9e27665f61517dc5fc097ea98 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:26 +0800 Subject: scripts/kernel-doc.py: adjust some coding style issues Make pylint happier by adding some missing documentation and addressing a couple of pylint warnings. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/0f9d5473105e4c09c6c41e3db72cc63f1d4d55f9.1744106242.git.mchehab+huawei@kernel.org --- scripts/lib/kdoc/kdoc_files.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) mode change 100755 => 100644 scripts/lib/kdoc/kdoc_files.py (limited to 'scripts/lib/kdoc/kdoc_files.py') diff --git a/scripts/lib/kdoc/kdoc_files.py b/scripts/lib/kdoc/kdoc_files.py old mode 100755 new mode 100644 index dd3dbe87520b..e2221db7022a --- a/scripts/lib/kdoc/kdoc_files.py +++ b/scripts/lib/kdoc/kdoc_files.py @@ -4,8 +4,6 @@ # # pylint: disable=R0903,R0913,R0914,R0917 -# TODO: implement warning filtering - """ Parse lernel-doc tags on multiple kernel source files. """ @@ -128,7 +126,7 @@ class KernelFiles(): def __init__(self, verbose=False, out_style=None, werror=False, wreturn=False, wshort_desc=False, wcontents_before_sections=False, - logger=None, modulename=None, export_file=None): + logger=None, modulename=None): """ Initialize startup variables and parse all files """ -- cgit From 2ab867a4941de2e9d7804e76ab002ad74c73b078 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:28 +0800 Subject: scripts/kernel-doc.py: move modulename to man class Only man output requires a modulename. Move its definition to the man class. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/583085e3885b0075d16ef9961b4f2ad870f30a55.1744106242.git.mchehab+huawei@kernel.org --- scripts/lib/kdoc/kdoc_files.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'scripts/lib/kdoc/kdoc_files.py') diff --git a/scripts/lib/kdoc/kdoc_files.py b/scripts/lib/kdoc/kdoc_files.py index e2221db7022a..5a6e92e34d05 100644 --- a/scripts/lib/kdoc/kdoc_files.py +++ b/scripts/lib/kdoc/kdoc_files.py @@ -126,7 +126,7 @@ class KernelFiles(): def __init__(self, verbose=False, out_style=None, werror=False, wreturn=False, wshort_desc=False, wcontents_before_sections=False, - logger=None, modulename=None): + logger=None): """ Initialize startup variables and parse all files """ @@ -134,9 +134,6 @@ class KernelFiles(): if not verbose: verbose = bool(os.environ.get("KBUILD_VERBOSE", 0)) - if not modulename: - modulename = "Kernel API" - if out_style is None: out_style = OutputFormat() @@ -168,7 +165,6 @@ class KernelFiles(): self.config.wreturn = wreturn self.config.wshort_desc = wshort_desc self.config.wcontents_before_sections = wcontents_before_sections - self.config.modulename = modulename self.config.function_table = set() self.config.source_map = {} -- cgit From 91d00bd54f300b614d48002d4ec8cc28b3f0b2a5 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:29 +0800 Subject: scripts/kernel-doc.py: properly handle KBUILD_BUILD_TIMESTAMP The logic that handles KBUILD_BUILD_TIMESTAMP is wrong, and adds a dependency of a third party module (dateutil). Fix it. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/ffc70a1b741b010365ed82f31611018f24f91ce7.1744106242.git.mchehab+huawei@kernel.org --- scripts/lib/kdoc/kdoc_files.py | 9 --------- 1 file changed, 9 deletions(-) (limited to 'scripts/lib/kdoc/kdoc_files.py') diff --git a/scripts/lib/kdoc/kdoc_files.py b/scripts/lib/kdoc/kdoc_files.py index 5a6e92e34d05..e52a6d05237e 100644 --- a/scripts/lib/kdoc/kdoc_files.py +++ b/scripts/lib/kdoc/kdoc_files.py @@ -13,9 +13,6 @@ import logging import os import re import sys -from datetime import datetime - -from dateutil import tz from kdoc_parser import KernelDoc from kdoc_output import OutputFormat @@ -137,12 +134,6 @@ class KernelFiles(): if out_style is None: out_style = OutputFormat() - dt = datetime.now() - if os.environ.get("KBUILD_BUILD_TIMESTAMP", None): - # use UTC TZ - to_zone = tz.gettz('UTC') - dt = dt.astimezone(to_zone) - if not werror: kcflags = os.environ.get("KCFLAGS", None) if kcflags: -- cgit From 11afeab6d74d1be80420b47113c4893c88dcc04b Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:31 +0800 Subject: scripts/kernel-doc.py: Properly handle Werror and exit codes The original kernel-doc script has a logic to return warnings as errors, and to report the number of warnings found, if in verbose mode. Implement it to be fully compatible with the original script. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/de33b0cebd9fdf82d8b221bcfe41db7269286222.1744106242.git.mchehab+huawei@kernel.org --- scripts/lib/kdoc/kdoc_files.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'scripts/lib/kdoc/kdoc_files.py') diff --git a/scripts/lib/kdoc/kdoc_files.py b/scripts/lib/kdoc/kdoc_files.py index e52a6d05237e..182d9ed58a72 100644 --- a/scripts/lib/kdoc/kdoc_files.py +++ b/scripts/lib/kdoc/kdoc_files.py @@ -12,7 +12,6 @@ import argparse import logging import os import re -import sys from kdoc_parser import KernelDoc from kdoc_output import OutputFormat @@ -109,7 +108,7 @@ class KernelFiles(): KernelDoc.process_export(self.config.function_table, line) except IOError: - print(f"Error: Cannot open fname {fname}", fname=sys.stderr) + self.config.log.error("Error: Cannot open fname %s", fname) self.config.errors += 1 def file_not_found_cb(self, fname): @@ -262,3 +261,12 @@ class KernelFiles(): fname, ln, dtype) if msg: yield fname, msg + + @property + def errors(self): + """ + Return a count of the number of warnings found, including + the ones displayed while interacting over self.msg. + """ + + return self.config.errors -- cgit From a566ba5af59524a3bc5cdfb46b248bd70a0972e9 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:33 +0800 Subject: scripts/lib/kdoc/kdoc_files.py: allow filtering output per fname For kerneldoc Sphinx extension, it is useful to display parsed results only from a single file. Change the logic at KernelFiles.msg() to allow such usage. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/9f5c0ff2568f34532ca99465fb378241d831d39f.1744106242.git.mchehab+huawei@kernel.org --- scripts/lib/kdoc/kdoc_files.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'scripts/lib/kdoc/kdoc_files.py') diff --git a/scripts/lib/kdoc/kdoc_files.py b/scripts/lib/kdoc/kdoc_files.py index 182d9ed58a72..527ab9117268 100644 --- a/scripts/lib/kdoc/kdoc_files.py +++ b/scripts/lib/kdoc/kdoc_files.py @@ -95,7 +95,7 @@ class KernelFiles(): doc = KernelDoc(self.config, fname) doc.run() - return doc + return doc.entries def process_export_file(self, fname): """ @@ -173,7 +173,7 @@ class KernelFiles(): # Initialize internal variables self.config.errors = 0 - self.results = [] + self.results = {} self.files = set() self.export_files = set() @@ -189,13 +189,9 @@ class KernelFiles(): # avoid reporting errors multiple times for fname in glob.parse_files(file_list, self.file_not_found_cb): - if fname in self.files: - continue - - res = self.parse_file(fname) - - self.results.append((res.fname, res.entries)) - self.files.add(fname) + if fname not in self.files: + self.results[fname] = self.parse_file(fname) + self.files.add(fname) # If a list of export files was provided, parse EXPORT_SYMBOL* # from files that weren't fully parsed @@ -226,7 +222,8 @@ class KernelFiles(): return self.out_style.msg(fname, name, arg) def msg(self, enable_lineno=False, export=False, internal=False, - symbol=None, nosymbol=None, no_doc_sections=False): + symbol=None, nosymbol=None, no_doc_sections=False, + filenames=None): """ Interacts over the kernel-doc results and output messages, returning kernel-doc markups on each interaction @@ -248,9 +245,12 @@ class KernelFiles(): function_table, enable_lineno, no_doc_sections) - for fname, arg_tuple in self.results: + if not filenames: + filenames = sorted(self.results.keys()) + + for fname in filenames: msg = "" - for name, arg in arg_tuple: + for name, arg in self.results[fname]: msg += self.out_msg(fname, name, arg) if msg is None: -- cgit From 16740c29dbf3275a22691d3d7c63701992872898 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:34 +0800 Subject: scripts/kernel_doc.py: better handle exported symbols Change the logic which detects internal/external symbols in a way that we can re-use it when calling via Sphinx extension. While here, remove an unused self.config var and let it clearer that self.config variables are read-only. This helps to allow handling multiple times in parallel if ever needed. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/6a69ba8d2b7ee6a6427abb53e60d09bd4d3565ee.1744106242.git.mchehab+huawei@kernel.org --- scripts/lib/kdoc/kdoc_files.py | 140 ++++++++++++++++++++++------------------- 1 file changed, 75 insertions(+), 65 deletions(-) (limited to 'scripts/lib/kdoc/kdoc_files.py') diff --git a/scripts/lib/kdoc/kdoc_files.py b/scripts/lib/kdoc/kdoc_files.py index 527ab9117268..dd003feefd1b 100644 --- a/scripts/lib/kdoc/kdoc_files.py +++ b/scripts/lib/kdoc/kdoc_files.py @@ -68,6 +68,9 @@ class GlobSourceFiles: handling directories if any """ + if not file_list: + return + for fname in file_list: if self.srctree: f = os.path.join(self.srctree, fname) @@ -84,40 +87,70 @@ class GlobSourceFiles: class KernelFiles(): """ - Parse lernel-doc tags on multiple kernel source files. + Parse kernel-doc tags on multiple kernel source files. + + There are two type of parsers defined here: + - self.parse_file(): parses both kernel-doc markups and + EXPORT_SYMBOL* macros; + - self.process_export_file(): parses only EXPORT_SYMBOL* macros. """ + def warning(self, msg): + """Ancillary routine to output a warning and increment error count""" + + self.config.log.warning(msg) + self.errors += 1 + + def error(self, msg): + """Ancillary routine to output an error and increment error count""" + + self.config.log.error(msg) + self.errors += 1 + def parse_file(self, fname): """ Parse a single Kernel source. """ + # Prevent parsing the same file twice if results are cached + if fname in self.files: + return + doc = KernelDoc(self.config, fname) - doc.run() + export_table, entries = doc.parse_kdoc() + + self.export_table[fname] = export_table + + self.files.add(fname) + self.export_files.add(fname) # parse_kdoc() already check exports - return doc.entries + self.results[fname] = entries def process_export_file(self, fname): """ Parses EXPORT_SYMBOL* macros from a single Kernel source file. """ - try: - with open(fname, "r", encoding="utf8", - errors="backslashreplace") as fp: - for line in fp: - KernelDoc.process_export(self.config.function_table, line) - except IOError: - self.config.log.error("Error: Cannot open fname %s", fname) - self.config.errors += 1 + # Prevent parsing the same file twice if results are cached + if fname in self.export_files: + return + + doc = KernelDoc(self.config, fname) + export_table = doc.parse_export() + + if not export_table: + self.error(f"Error: Cannot check EXPORT_SYMBOL* on {fname}") + export_table = set() + + self.export_table[fname] = export_table + self.export_files.add(fname) def file_not_found_cb(self, fname): """ Callback to warn if a file was not found. """ - self.config.log.error("Cannot find file %s", fname) - self.config.errors += 1 + self.error(f"Cannot find file {fname}") def __init__(self, verbose=False, out_style=None, werror=False, wreturn=False, wshort_desc=False, @@ -147,7 +180,9 @@ class KernelFiles(): if kdoc_werror: werror = kdoc_werror - # Set global config data used on all files + # Some variables are global to the parser logic as a whole as they are + # used to send control configuration to KernelDoc class. As such, + # those variables are read-only inside the KernelDoc. self.config = argparse.Namespace self.config.verbose = verbose @@ -156,27 +191,25 @@ class KernelFiles(): self.config.wshort_desc = wshort_desc self.config.wcontents_before_sections = wcontents_before_sections - self.config.function_table = set() - self.config.source_map = {} - if not logger: self.config.log = logging.getLogger("kernel-doc") else: self.config.log = logger - self.config.kernel_version = os.environ.get("KERNELVERSION", - "unknown kernel version'") + self.config.warning = self.warning + self.config.src_tree = os.environ.get("SRCTREE", None) - self.out_style = out_style + # Initialize variables that are internal to KernelFiles - # Initialize internal variables + self.out_style = out_style - self.config.errors = 0 + self.errors = 0 self.results = {} self.files = set() self.export_files = set() + self.export_table = {} def parse(self, file_list, export_file=None): """ @@ -185,28 +218,11 @@ class KernelFiles(): glob = GlobSourceFiles(srctree=self.config.src_tree) - # Prevent parsing the same file twice to speedup parsing and - # avoid reporting errors multiple times - for fname in glob.parse_files(file_list, self.file_not_found_cb): - if fname not in self.files: - self.results[fname] = self.parse_file(fname) - self.files.add(fname) - - # If a list of export files was provided, parse EXPORT_SYMBOL* - # from files that weren't fully parsed - - if not export_file: - return - - self.export_files |= self.files - - glob = GlobSourceFiles(srctree=self.config.src_tree) + self.parse_file(fname) for fname in glob.parse_files(export_file, self.file_not_found_cb): - if fname not in self.export_files: - self.process_export_file(fname) - self.export_files.add(fname) + self.process_export_file(fname) def out_msg(self, fname, name, arg): """ @@ -223,32 +239,35 @@ class KernelFiles(): def msg(self, enable_lineno=False, export=False, internal=False, symbol=None, nosymbol=None, no_doc_sections=False, - filenames=None): + filenames=None, export_file=None): """ Interacts over the kernel-doc results and output messages, returning kernel-doc markups on each interaction """ - function_table = self.config.function_table - - if symbol: - for s in symbol: - function_table.add(s) - - # Output none mode: only warnings will be shown - if not self.out_style: - return - self.out_style.set_config(self.config) - self.out_style.set_filter(export, internal, symbol, nosymbol, - function_table, enable_lineno, - no_doc_sections) - if not filenames: filenames = sorted(self.results.keys()) for fname in filenames: + function_table = set() + + if internal or export: + if not export_file: + export_file = [fname] + + for f in export_file: + function_table |= self.export_table[f] + + if symbol: + for s in symbol: + function_table.add(s) + + self.out_style.set_filter(export, internal, symbol, nosymbol, + function_table, enable_lineno, + no_doc_sections) + msg = "" for name, arg in self.results[fname]: msg += self.out_msg(fname, name, arg) @@ -261,12 +280,3 @@ class KernelFiles(): fname, ln, dtype) if msg: yield fname, msg - - @property - def errors(self): - """ - Return a count of the number of warnings found, including - the ones displayed while interacting over self.msg. - """ - - return self.config.errors -- cgit