scripts: test_doc_build.py: make capture assynchronous

Prepare the tool to allow writing the output into log files.
For such purpose, receive stdin/stdout messages asynchronously.

Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
Signed-off-by: Jonathan Corbet <corbet@lwn.net>
Link: https://lore.kernel.org/r/9b0a60b5047137b5ba764701268da992767b128c.1750571906.git.mchehab+huawei@kernel.org

1 files changed, 230 insertions, 113 deletions

diff --git a/scripts/test_doc_build.py b/scripts/test_doc_build.py
index 482716fbe91d..94f2f2d8c3b7 100755
--- a/scripts/test_doc_build.py
+++ b/scripts/test_doc_build.py
@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: GPL-2.0
 # Copyright(c) 2025: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
 #
-# pylint: disable=C0103,R1715
+# pylint: disable=R0903,R0913,R0914,R0917
 
 """
 Install minimal supported requirements for different Sphinx versions
@@ -10,20 +10,20 @@ and optionally test the build.
 """
 
 import argparse
+import asyncio
 import os.path
 import sys
 import time
-
-from subprocess import run
+import subprocess
 
 # Minimal python version supported by the building system
-python_bin = "python3.9"
+MINIMAL_PYTHON_VERSION = "python3.9"
 
 # Starting from 8.0.2, Python 3.9 becomes too old
-python_changes = {(8, 0, 2): "python3"}
+PYTHON_VER_CHANGES = {(8, 0, 2): "python3"}
 
 # Sphinx versions to be installed and their incremental requirements
-sphinx_requirements = {
+SPHINX_REQUIREMENTS = {
     (3, 4, 3): {
         "alabaster": "0.7.13",
         "babel": "2.17.0",
@@ -101,141 +101,258 @@ sphinx_requirements = {
 }
 
 
-def parse_version(ver_str):
-    """Convert a version string into a tuple."""
+class AsyncCommands:
+    """Excecute command synchronously"""
+
+    stdout = None
+    stderr = None
+    output = None
+
+    async def _read(self, stream, verbose, is_info):
+        """Ancillary routine to capture while displaying"""
+
+        while stream is not None:
+            line = await stream.readline()
+            if line:
+                out = line.decode("utf-8", errors="backslashreplace")
+                self.output += out
+                if is_info:
+                    if verbose:
+                        print(out.rstrip("\n"))
+
+                    self.stdout += out
+                else:
+                    if verbose:
+                        print(out.rstrip("\n"), file=sys.stderr)
+
+                    self.stderr += out
+            else:
+                break
+
+    async def run(self, cmd, capture_output=False, check=False,
+                  env=None, verbose=True):
+
+        """
+        Execute an arbitrary command, handling errors.
+
+        Please notice that this class is not thread safe
+        """
+
+        self.stdout = ""
+        self.stderr = ""
+        self.output = ""
+
+        if verbose:
+            print("$ ", " ".join(cmd))
+
+        proc = await asyncio.create_subprocess_exec(cmd[0],
+                                                    *cmd[1:],
+                                                    env=env,
+                                                    stdout=asyncio.subprocess.PIPE,
+                                                    stderr=asyncio.subprocess.PIPE)
+
+        # Handle input and output in realtime
+        await asyncio.gather(
+            self._read(proc.stdout, verbose, True),
+            self._read(proc.stderr, verbose, False),
+        )
+
+        await proc.wait()
+
+        if check and proc.returncode > 0:
+            raise subprocess.CalledProcessError(returncode=proc.returncode,
+                                                cmd=" ".join(cmd),
+                                                output=self.stdout,
+                                                stderr=self.stderr)
+
+        if capture_output:
+            if proc.returncode > 0:
+                print("Error {proc.returncode}", file=sys.stderr)
+                return ""
+
+            return self.output
+
+        ret = subprocess.CompletedProcess(args=cmd,
+                                          returncode=proc.returncode,
+                                          stdout=self.stdout,
+                                          stderr=self.stderr)
+
+        return ret
 
-    return tuple(map(int, ver_str.split(".")))
 
+class SphinxVenv:
+    """
+    Installs Sphinx on one virtual env per Sphinx version with a minimal
+    set of dependencies, adjusting them to each specific version.
+    """
 
-parser = argparse.ArgumentParser(description="Build docs for different sphinx_versions.")
+    def __init__(self):
+        """Initialize instance variables"""
 
-parser.add_argument('-v', '--version', help='Sphinx single version',
-                    type=parse_version)
-parser.add_argument('--min-version', "--min", help='Sphinx minimal version',
-                    type=parse_version)
-parser.add_argument('--max-version', "--max", help='Sphinx maximum version',
-                    type=parse_version)
-parser.add_argument('-a', '--make_args',
-                    help='extra arguments for make htmldocs, like SPHINXDIRS=netlink/specs',
-                    nargs="*")
-parser.add_argument('-w', '--write', help='write a requirements.txt file',
-                    action='store_true')
-parser.add_argument('-m', '--make',
-                    help='Make documentation',
-                    action='store_true')
-parser.add_argument('-i', '--wait-input',
-                    help='Wait for an enter before going to the next version',
-                    action='store_true')
+        self.built_time = {}
+        self.first_run = True
 
-args = parser.parse_args()
-
-if not args.make_args:
-    args.make_args = []
+    async def _handle_version(self, args, cur_ver, cur_requirements, python_bin):
+        """Handle a single Sphinx version"""
 
-if args.version:
-    if args.min_version or args.max_version:
-        sys.exit("Use either --version or --min-version/--max-version")
-    else:
-        args.min_version = args.version
-        args.max_version = args.version
+        cmd = AsyncCommands()
 
-sphinx_versions = sorted(list(sphinx_requirements.keys()))
+        ver = ".".join(map(str, cur_ver))
 
-if not args.min_version:
-    args.min_version = sphinx_versions[0]
+        if not self.first_run and args.wait_input and args.make:
+            ret = input("Press Enter to continue or 'a' to abort: ").strip().lower()
+            if ret == "a":
+                print("Aborted.")
+                sys.exit()
+        else:
+            self.first_run = False
 
-if not args.max_version:
-    args.max_version = sphinx_versions[-1]
+        venv_dir = f"Sphinx_{ver}"
+        req_file = f"requirements_{ver}.txt"
 
-first_run = True
-cur_requirements = {}
-built_time = {}
+        print(f"\nSphinx {ver} with {python_bin}")
 
-for cur_ver, new_reqs in sphinx_requirements.items():
-    cur_requirements.update(new_reqs)
+        # Create venv
+        await cmd.run([python_bin, "-m", "venv", venv_dir], check=True)
+        pip = os.path.join(venv_dir, "bin/pip")
 
-    if cur_ver in python_changes:
-        python_bin = python_changes[cur_ver]
+        # Create install list
+        reqs = []
+        for pkg, verstr in cur_requirements.items():
+            reqs.append(f"{pkg}=={verstr}")
 
-    ver = ".".join(map(str, cur_ver))
+        reqs.append(f"Sphinx=={ver}")
 
-    if args.min_version:
-        if cur_ver < args.min_version:
-            continue
+        await cmd.run([pip, "install"] + reqs, check=True, verbose=True)
 
-    if args.max_version:
-        if cur_ver > args.max_version:
-            break
+        # Freeze environment
+        result = await cmd.run([pip, "freeze"], verbose=False, check=True)
 
-    if not first_run and args.wait_input and args.make:
-        ret = input("Press Enter to continue or 'a' to abort: ").strip().lower()
-        if ret == "a":
-            print("Aborted.")
-            sys.exit()
-    else:
-        first_run = False
+        # Pip install succeeded. Write requirements file
+        if args.write:
+            with open(req_file, "w", encoding="utf-8") as fp:
+                fp.write(result.stdout)
 
-    venv_dir = f"Sphinx_{ver}"
-    req_file = f"requirements_{ver}.txt"
+        if args.make:
+            start_time = time.time()
 
-    print(f"\nSphinx {ver} with {python_bin}")
+            # Prepare a venv environment
+            env = os.environ.copy()
+            bin_dir = os.path.join(venv_dir, "bin")
+            env["PATH"] = bin_dir + ":" + env["PATH"]
+            env["VIRTUAL_ENV"] = venv_dir
+            if "PYTHONHOME" in env:
+                del env["PYTHONHOME"]
+
+            # Test doc build
+            await cmd.run(["make", "cleandocs"], env=env, check=True)
+            make = ["make"] + args.make_args + ["htmldocs"]
+
+            print(f". {bin_dir}/activate")
+            print(" ".join(make))
+            print("deactivate")
+            await cmd.run(make, env=env, check=True)
+
+            end_time = time.time()
+            elapsed_time = end_time - start_time
+            hours, minutes = divmod(elapsed_time, 3600)
+            minutes, seconds = divmod(minutes, 60)
+
+            hours = int(hours)
+            minutes = int(minutes)
+            seconds = int(seconds)
+
+            self.built_time[ver] = f"{hours:02d}:{minutes:02d}:{seconds:02d}"
+
+            print(f"Finished doc build for Sphinx {ver}. Elapsed time: {self.built_time[ver]}")
+
+    async def run(self, args):
+        """
+        Navigate though multiple Sphinx versions, handling each of them
+        on a loop.
+        """
+
+        cur_requirements = {}
+        python_bin = MINIMAL_PYTHON_VERSION
+
+        for cur_ver, new_reqs in SPHINX_REQUIREMENTS.items():
+            cur_requirements.update(new_reqs)
+
+            if cur_ver in PYTHON_VER_CHANGES:          # pylint: disable=R1715
+
+                python_bin = PYTHON_VER_CHANGES[cur_ver]
+
+            if args.min_version:
+                if cur_ver < args.min_version:
+                    continue
+
+            if args.max_version:
+                if cur_ver > args.max_version:
+                    break
+
+            await self._handle_version(args, cur_ver, cur_requirements,
+                                       python_bin)
+
+        if args.make:
+            print()
+            print("Summary:")
+            for ver, elapsed_time in sorted(self.built_time.items()):
+                print(f"\tSphinx {ver} elapsed time: {elapsed_time}")
+
+
+def parse_version(ver_str):
+    """Convert a version string into a tuple."""
+
+    return tuple(map(int, ver_str.split(".")))
 
-    # Create venv
-    run([python_bin, "-m", "venv", venv_dir], check=True)
-    pip = os.path.join(venv_dir, "bin/pip")
 
-    # Create install list
-    reqs = []
-    for pkg, verstr in cur_requirements.items():
-        reqs.append(f"{pkg}=={verstr}")
+async def main():
+    """Main program"""
 
-    reqs.append(f"Sphinx=={ver}")
+    parser = argparse.ArgumentParser(description="Build docs for different sphinx_versions.")
 
-    run([pip, "install"] + reqs, check=True)
+    parser.add_argument('-v', '--version', help='Sphinx single version',
+                        type=parse_version)
+    parser.add_argument('--min-version', "--min", help='Sphinx minimal version',
+                        type=parse_version)
+    parser.add_argument('--max-version', "--max", help='Sphinx maximum version',
+                        type=parse_version)
+    parser.add_argument('-a', '--make_args',
+                        help='extra arguments for make htmldocs, like SPHINXDIRS=netlink/specs',
+                        nargs="*")
+    parser.add_argument('-w', '--write', help='write a requirements.txt file',
+                        action='store_true')
+    parser.add_argument('-m', '--make',
+                        help='Make documentation',
+                        action='store_true')
+    parser.add_argument('-i', '--wait-input',
+                        help='Wait for an enter before going to the next version',
+                        action='store_true')
 
-    # Freeze environment
-    result = run([pip, "freeze"], capture_output=True, text=True, check=True)
+    args = parser.parse_args()
 
-    # Pip install succeeded. Write requirements file
-    if args.write:
-        with open(req_file, "w", encoding="utf-8") as fp:
-            fp.write(result.stdout)
+    if not args.make_args:
+        args.make_args = []
 
-    if args.make:
-        start_time = time.time()
+    if args.version:
+        if args.min_version or args.max_version:
+            sys.exit("Use either --version or --min-version/--max-version")
+        else:
+            args.min_version = args.version
+            args.max_version = args.version
 
-        # Prepare a venv environment
-        env = os.environ.copy()
-        bin_dir = os.path.join(venv_dir, "bin")
-        env["PATH"] = bin_dir + ":" + env["PATH"]
-        env["VIRTUAL_ENV"] = venv_dir
-        if "PYTHONHOME" in env:
-            del env["PYTHONHOME"]
-
-        # Test doc build
-        run(["make", "cleandocs"], env=env, check=True)
-        make = ["make"] + args.make_args + ["htmldocs"]
-
-        print(f". {bin_dir}/activate")
-        print(" ".join(make))
-        print("deactivate")
-        run(make, env=env, check=True)
+    sphinx_versions = sorted(list(SPHINX_REQUIREMENTS.keys()))
 
-        end_time = time.time()
-        elapsed_time = end_time - start_time
-        hours, minutes = divmod(elapsed_time, 3600)
-        minutes, seconds = divmod(minutes, 60)
+    if not args.min_version:
+        args.min_version = sphinx_versions[0]
 
-        hours = int(hours)
-        minutes = int(minutes)
-        seconds = int(seconds)
+    if not args.max_version:
+        args.max_version = sphinx_versions[-1]
 
-        built_time[ver] = f"{hours:02d}:{minutes:02d}:{seconds:02d}"
+    venv = SphinxVenv()
+    await venv.run(args)
 
-        print(f"Finished doc build for Sphinx {ver}. Elapsed time: {built_time[ver]}")
 
-if args.make:
-    print()
-    print("Summary:")
-    for ver, elapsed_time in sorted(built_time.items()):
-        print(f"\tSphinx {ver} elapsed time: {elapsed_time}")
+# Call main method
+if __name__ == "__main__":
+    asyncio.run(main())