From 59000e9e65899e97252dfd24ff88f5533e24aa46 Mon Sep 17 00:00:00 2001
From: Xavier Dupre <xadupre@microsoft.com>
Date: Mon, 25 Sep 2023 12:18:41 +0200
Subject: [PATCH 1/3] Add command to check the readme syntax

---
 _unittests/ut__main/test_readme.py |  28 ++
 sphinx_runpython/_cmd_helper.py    |  18 +-
 sphinx_runpython/readme.py         | 463 +++++++++++++++++++++++++++++
 3 files changed, 506 insertions(+), 3 deletions(-)
 create mode 100644 _unittests/ut__main/test_readme.py
 create mode 100644 sphinx_runpython/readme.py

diff --git a/_unittests/ut__main/test_readme.py b/_unittests/ut__main/test_readme.py
new file mode 100644
index 0000000..8c17961
--- /dev/null
+++ b/_unittests/ut__main/test_readme.py
@@ -0,0 +1,28 @@
+import os
+import unittest
+from io import StringIO
+from contextlib import redirect_stdout
+from tempfile import TemporaryDirectory
+from sphinx_runpython.ext_test_case import ExtTestCase
+from sphinx_runpython.readme import check_readme_syntax
+
+
+class TestReadme(ExtTestCase):
+    def test_venv_docutils08_readme(self):
+        fold = os.path.dirname(os.path.abspath(__file__))
+        readme = os.path.join(fold, "..", "..", "README.rst")
+        assert os.path.exists(readme)
+        with open(readme, "r", encoding="utf8") as f:
+            content = f.read()
+        self.assertNotEmpty(content)
+
+        with TemporaryDirectory() as temp:
+            st = StringIO()
+            with redirect_stdout(st):
+                check_readme_syntax(readme, folder=temp, verbose=1)
+            text = st.getvalue()
+            print(text)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/sphinx_runpython/_cmd_helper.py b/sphinx_runpython/_cmd_helper.py
index 7dab12b..0817d5c 100644
--- a/sphinx_runpython/_cmd_helper.py
+++ b/sphinx_runpython/_cmd_helper.py
@@ -1,6 +1,7 @@
 import glob
 import os
 from argparse import ArgumentParser
+from tempfile import TemporaryDirectory
 from .convert import convert_ipynb_to_gallery
 
 
@@ -10,12 +11,17 @@ def get_parser():
         description="A collection of quick tools.",
         epilog="",
     )
-    parser.add_argument("command", help="Command to run, only nb2py is available")
     parser.add_argument(
-        "-p", "--path", help="Folder which contains the files to process"
+        "command", help="Command to run, only 'nb2py' or 'readme' are available"
     )
     parser.add_argument(
-        "-r", "--recursive", help="Recursive search.", action="store_true"
+        "-p", "--path", help="Folder or file which contains the files to process"
+    )
+    parser.add_argument(
+        "-r",
+        "--recursive",
+        help="Recursive search.",
+        action="store_true",
     )
     parser.add_argument("-v", "--verbose", help="verbosity", default=1, type=int)
     return parser
@@ -41,6 +47,12 @@ def process_args(args):
     if cmd == "nb2py":
         nb2py(args.path, recursive=args.recursive, verbose=args.verbose)
         return
+    if cmd == "readme":
+        from .readme import check_readme_syntax
+
+        with TemporaryDirectory() as temp:
+            check_readme_syntax(args.path, verbose=args.verbose, folder=temp)
+        return
     raise ValueError(f"Command {cmd!r} is unknown.")
 
 
diff --git a/sphinx_runpython/readme.py b/sphinx_runpython/readme.py
new file mode 100644
index 0000000..ad7aea7
--- /dev/null
+++ b/sphinx_runpython/readme.py
@@ -0,0 +1,463 @@
+import os
+import sys
+from typing import Any, Dict, List, Optional
+from urllib.request import urlretrieve
+
+
+class VirtualEnvError(Exception):
+    """
+    Exception raised by the function implemented in this file.
+    """
+
+    pass
+
+
+class NotImplementedErrorFromVirtualEnvironment(NotImplementedError):
+    """
+    Defines an exception when a function does not work
+    in a virtual environment.
+    """
+
+    pass
+
+
+def is_virtual_environment() -> bool:
+    """
+    Tells if the script is run from a virtual environment.
+    """
+    return (
+        getattr(sys, "base_exec_prefix", sys.exec_prefix) != sys.exec_prefix
+    ) or hasattr(sys, "real_prefix")
+
+
+def build_venv_cmd(params: Dict[str, Any], posparams: List[Any]) -> str:
+    """
+    Builds the command line for virtual env.
+
+    :param params: dictionary of parameters
+    :param posparams: positional arguments
+    :return: string
+    """
+    import venv
+
+    v = venv.__file__
+    if v is None:
+        raise ImportError("module venv should have a version number")
+    exe = sys.executable.replace("w.exe", "").replace(".exe", "")
+    cmd = [exe, "-m", "venv"]
+    for k, v in params.items():
+        if v is None:
+            cmd.append("--" + k)
+        else:
+            cmd.append("--" + k + "=" + v)
+    cmd.extend(posparams)
+    return " ".join(cmd)
+
+
+def create_virtual_env(
+    where: str,
+    symlinks: bool = False,
+    system_site_packages: bool = False,
+    clear: bool = True,
+    packages: Optional[List[str]] = None,
+    verbose: int = 0,
+    temp_folder: Optional[str] = None,
+    platform: Optional[str] = None,
+) -> str:
+    """
+    Creates a virtual environment.
+
+    :param where: location of this virtual environment
+    :param symlinks: attempt to symlink rather than copy
+    :param system_site_packages: Give the virtual environment
+        access to the system site-packages dir
+    :param clear: Delete the environment directory if it already exists.
+        If not specified and the directory exists, an error is raised.
+    :param packages: list of packages to install
+    :param temp_folder: temporary folder (to download module if needed),
+        by default ``<where>/download``
+    :param platform: platform to use
+    :param verbose: verbosity
+    :return: standard output
+
+    .. faqref::
+        :title: How to create a virtual environment?
+
+        The following example creates a virtual environment.
+        Packages can be added by specifying the parameter *package*.
+
+        ::
+
+            import os
+            from sphinx_runpython.readme import create_virtual_env
+
+            fold = "my_env"
+            if not os.path.exists(fold):
+                os.mkdir(fold)
+            create_virtual_env(fold)
+
+    The function does not work from a virtual environment.
+    """
+    from .runpython import run_cmd
+
+    if is_virtual_environment():
+        raise NotImplementedErrorFromVirtualEnvironment()
+
+    if verbose > 0:
+        print(f"[create_virtual_env] create virtual environment at {where!r}")
+    params = {}
+    if symlinks:
+        params["symlinks"] = None
+    if system_site_packages:
+        params["system-site-packages"] = None
+    if clear:
+        params["clear"] = None
+    cmd = build_venv_cmd(params, [where])
+    out, err = run_cmd(cmd, wait=True, logf=print if verbose else None)
+    if len(err) > 0:
+        raise VirtualEnvError(
+            f"Unable to create virtual environement at {where!r}"
+            f"\nCMD:\n{cmd}\nOUT:\n{out}\n[pyqerror]\n{err}"
+        )
+
+    if platform is None:
+        platform = sys.platform
+    if platform.startswith("win"):
+        scripts = os.path.join(where, "Scripts")
+    else:
+        scripts = os.path.join(where, "bin")
+
+    if not os.path.exists(scripts):
+        files = "\n  ".join(os.listdir(where))
+        raise FileNotFoundError(f"Unable to find {files}, content:\n  {scripts}")
+
+    in_scripts = os.listdir(scripts)
+    pips = [_ for _ in in_scripts if _.startswith("pip")]
+    if len(pips) == 0:
+        out += venv_install(
+            where, "pip", verbose=verbose, temp_folder=temp_folder, platform=platform
+        )
+    in_scripts = os.listdir(scripts)
+    pips = [_ for _ in in_scripts if _.startswith("pip")]
+    if len(pips) == 0:
+        raise FileNotFoundError(
+            f"Unable to find pip in {in_scripts!r}, content:\n  {scripts}"
+        )
+
+    out += venv_install(
+        where, "pip", verbose=verbose, temp_folder=temp_folder, platform=platform
+    )
+
+    if packages is not None and len(packages) > 0:
+        if verbose > 0:
+            print(f"[create_virtual_env] install packages in {where}")
+        packages = [_ for _ in packages if _ not in ("pip",)]
+        if len(packages) > 0:
+            out += venv_install(
+                where,
+                packages,
+                verbose=verbose,
+                temp_folder=temp_folder,
+                platform=platform,
+            )
+    return out
+
+
+def venv_install(
+    venv: str,
+    packages: List[str],
+    verbose: int = 0,
+    temp_folder: Optional[str] = None,
+    platform: Optional[str] = None,
+) -> str:
+    """
+    Installs a package or a list of packages in a virtual environment.
+
+    :param venv: location of the virtual environment
+    :param packages: a package (str) or a list of packages(list[str])
+    :param temp_folder: temporary folder (to download module if needed),
+        by default ``<where>/download``
+    :param platform: platform (``sys.platform`` by default)
+    :param verbose: verbosity
+    :return: standard output
+
+    The function does not work from a virtual environment.
+    """
+    from .runpython import run_cmd
+
+    if is_virtual_environment():
+        raise NotImplementedErrorFromVirtualEnvironment()
+    if temp_folder is None:
+        temp_folder = os.path.join(venv, "download")
+    if isinstance(packages, str):
+        packages = [packages]
+    if platform is None:
+        platform = sys.platform
+
+    exe = os.path.join(venv, "bin", "python")
+    get_pip = os.path.join(venv, "get_pip.py")
+    if packages == "pip" or packages == ["pip"]:
+        if not os.path.exists(get_pip):
+            if verbose > 2:
+                print("[bench_virtual] install pip")
+            urlretrieve("https://bootstrap.pypa.io/get-pip.py", get_pip)
+        cmd = [exe, get_pip]
+        out, err = run_cmd([exe, get_pip], wait=True)
+    else:
+        pcks = " ".join(packages)
+        cmd = f"{exe} -m pip install {pcks}"
+        out, err = run_cmd(cmd, wait=True)
+
+    lines = [
+        _
+        for _ in err.split("\n")
+        if "requires" not in _
+        and "pip's dependency resolver does not currently " not in _
+    ]
+    err = "\n".join(lines)
+    if len(err) > 0:
+        raise RuntimeError(
+            f"Unable to run cmd={cmd!r} in {venv!r} "
+            f"(path={get_pip!r}) due to\n{err}"
+        )
+    if verbose > 2:
+        print(out)
+    return out
+
+
+def run_venv_script(
+    venv: str,
+    script: str,
+    verbose: int = 0,
+    is_file: bool = False,
+    is_cmd: bool = False,
+    skip_err_if: Optional[bool] = None,
+    platform: Optional[str] = None,
+    **kwargs: Dict[str, Any],
+) -> str:
+    """
+    Runs a script on a vritual environment (the script should be simple).
+
+    :param venv: virtual environment
+    :param script: script as a string (not a file)
+    :param is_file: is script a file or a string to execute
+    :param is_cmd: if True, script is a command line to run
+        (as a list) for python executable
+    :param skip_err_if: do not pay attention to standard
+        error if this string was found in standard output
+    :param platform: platform (``sys.platform`` by default)
+    :param verbose: verbosity
+    :param kwargs: others arguments for function @see fn run_cmd.
+    :return: output
+
+    The function does not work from a virtual environment.
+    """
+    from .runpython import run_cmd
+
+    def filter_err(err):
+        lis = err.split("\n")
+        lines = []
+        for li in lis:
+            if "missing dependencies" in li:
+                continue
+            if "' misses '" in li:
+                continue
+            lines.append(li)
+        return "\n".join(lines).strip(" \r\n\t")
+
+    if is_virtual_environment():
+        raise NotImplementedErrorFromVirtualEnvironment()
+
+    if platform is None:
+        platform = sys.platform
+
+    if platform.startswith("win"):
+        exe = os.path.join(venv, "Scripts", "python")
+    else:
+        exe = os.path.join(venv, "bin", "python")
+    if is_cmd:
+        cmd = " ".join([exe] + script)
+        out, err = run_cmd(cmd, wait=True, logf=print if verbose else None, **kwargs)
+        err = filter_err(err)
+        if len(err) > 0 and (skip_err_if is None or skip_err_if not in out):
+            raise VirtualEnvError(
+                "unable to run cmd at {2}\n--CMD--\n{3}\n--OUT--\n{0}\n[pyqerror]"
+                "\n{1}".format(out, err, venv, cmd)
+            )
+        return out
+
+    script = ";".join(script.split("\n"))
+    if is_file:
+        if not os.path.exists(script):
+            raise FileNotFoundError(script)
+        cmd = " ".join([exe, "-u", '"{0}"'.format(script)])
+    else:
+        cmd = " ".join([exe, "-u", "-c", '"{0}"'.format(script)])
+    out, err = run_cmd(cmd, wait=True, logf=print if verbose else None, **kwargs)
+    err = filter_err(err)
+    if len(err) > 0:
+        raise VirtualEnvError(
+            f"Unable to run script at {venv!r}\n--CMD--\n{cmd}\n--OUT--\n{out}\n"
+            f"[pyqerror]\n{err}"
+        )
+    return out
+
+
+def run_base_script(
+    script: str,
+    is_file: bool = False,
+    is_cmd: bool = False,
+    verbose: int = 0,
+    skip_err_if: Optional[bool] = None,
+    argv: Optional[List[str]] = None,
+    platform: Optional[str] = None,
+    **kwargs: Dict[str, Any],
+) -> str:
+    """
+    Runs a script with the original intepreter even if this function
+    is run from a virtual environment.
+
+    :param script: script as a string (not a file)
+    :param is_file: is script a file or a string to execute
+    :param is_cmd: if True, script is a command line to run
+        (as a list) for python executable
+    :param skip_err_if: do not pay attention to standard error
+        if this string was found in standard output
+    :param argv: list of arguments to add on the command line
+    :param platform: platform (``sys.platform`` by default)
+    :param kwargs: others arguments for function @see fn run_cmd.
+    :param verbose: verbosity
+    :return: output
+
+    The function does not work from a virtual environment.
+    The function does not raise an exception if the standard error
+    contains something like::
+
+        ----------------------------------------------------------------------
+        Ran 1 test in 0.281s
+
+        OK
+    """
+    from ..loghelper import run_cmd
+
+    def true_err(err):
+        if "Ran 1 test" in err and "OK" in err:
+            return False
+        return True
+
+    if platform is None:
+        platform = sys.platform
+
+    if hasattr(sys, "real_prefix"):
+        exe = sys.base_prefix
+    elif hasattr(sys, "base_exec_prefix"):
+        exe = sys.base_exec_prefix
+    else:
+        exe = sys.exec_prefix
+
+    if platform.startswith("win"):
+        exe = os.path.join(exe, "python")
+    else:
+        exe = os.path.join(exe, "bin", "python%d.%d" % sys.version_info[:2])
+        if not os.path.exists(exe):
+            exe = os.path.join(exe, "bin", "python")
+
+    if is_cmd:
+        cmd = " ".join([exe] + script)
+        if argv is not None:
+            cmd += " " + " ".join(argv)
+        out, err = run_cmd(cmd, wait=True, verbose=verbose, **kwargs)
+        if (
+            len(err) > 0
+            and (skip_err_if is None or skip_err_if not in out)
+            and true_err(err)
+        ):
+            p = sys.base_prefix if hasattr(sys, "base_prefix") else sys.prefix
+            raise VirtualEnvError(
+                f"Unable to run cmd at {p!r}\nCMD:\n{cmd}"
+                f"\nOUT:\n{out}\n[pyqerror]\n{err}"
+            )
+        return out
+
+    script = ";".join(script.split("\n"))
+    if is_file:
+        if not os.path.exists(script):
+            raise FileNotFoundError(script)
+        cmd = " ".join([exe, "-u", '"{0}"'.format(script)])
+    else:
+        cmd = " ".join([exe, "-u", "-c", '"{0}"'.format(script)])
+    if argv is not None:
+        cmd += " " + " ".join(argv)
+    out, err = run_cmd(cmd, wait=True, verbose=verbose, **kwargs)
+    if len(err) > 0 and true_err(err):
+        p = sys.base_prefix if hasattr(sys, "base_prefix") else sys.prefix
+        raise VirtualEnvError(
+            f"Unable to run script at {p!r}\nCMD:\n{cmd}"
+            f"\nOUT:\n{out}\n[pyqerror]\n{err}"
+        )
+    return out
+
+
+def check_readme_syntax(
+    readme: str, folder: str, version: Optional[str] = None, verbose: int = 0
+) -> str:
+    """
+    Checks the syntax of the file ``readme.rst``
+    which describes a python project.
+
+    :param readme: file to check
+    :param folder: location for the virtual environment
+    :param version: version of docutils
+    :param verbose: verbosity
+    :return: output or SyntaxError exception
+    """
+    if is_virtual_environment():
+        raise NotImplementedErrorFromVirtualEnvironment()
+    if not os.path.exists(folder):
+        os.makedirs(folder)
+
+    out = create_virtual_env(
+        folder,
+        verbose=verbose,
+        packages=["docutils" if version is None else f"docutils=={version}"],
+    )
+    outfile = os.path.join(folder, "conv_readme.html")
+
+    script = [
+        "from docutils import core",
+        "import io",
+        "from docutils.readers.standalone import Reader",
+        "from docutils.parsers.rst import Parser",
+        "from docutils.parsers.rst.directives.images import Image",
+        "from docutils.parsers.rst.directives import _directives",
+        "from docutils.writers.html4css1 import Writer",
+        "_directives['image'] = Image",
+        "with open('{0}', 'r', encoding='utf8') as g: s = g.read()".format(
+            readme.replace("\\", "\\\\")
+        ),
+        "settings_overrides = {'output_encoding': 'unicode', 'doctitle_xform': True,",
+        "            'initial_header_level': 2, 'warning_stream': io.StringIO()}",
+        "parts = core.publish_parts(source=s, parser=Parser(), "
+        "            reader=Reader(), source_path=None,",
+        "            destination_path=None, writer=Writer(),",
+        "            settings_overrides=settings_overrides)",
+        "with open('{0}', 'w', encoding='utf8') as f: f.write(parts['whole'])".format(
+            outfile.replace("\\", "\\\\")
+        ),
+    ]
+
+    file_script = os.path.join(folder, "test_" + os.path.split(readme)[-1])
+    with open(file_script, "w") as f:
+        f.write("\n".join(script))
+
+    out = run_venv_script(folder, file_script, verbose=verbose, is_file=True)
+    with open(outfile, "r", encoding="utf8") as h:
+        content = h.read()
+
+    if "System Message" in content:
+        raise SyntaxError(
+            f"Unable to parse a file with docutils=={version!r}"
+            f"\n------\n{out}\n------\nCONTENT:\n{content}"
+        )
+
+    return out

From f55b042805b1e31c5b0099513f2c99f2ec87ac1f Mon Sep 17 00:00:00 2001
From: Xavier Dupre <xadupre@microsoft.com>
Date: Mon, 25 Sep 2023 12:26:21 +0200
Subject: [PATCH 2/3] documentation

---
 CHANGELOGS.rst                     |  1 +
 _doc/api/tools.rst                 | 19 ++++++++++++++++++-
 _unittests/ut__main/test_readme.py |  4 ++++
 3 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/CHANGELOGS.rst b/CHANGELOGS.rst
index 97646d8..c19c718 100644
--- a/CHANGELOGS.rst
+++ b/CHANGELOGS.rst
@@ -4,6 +4,7 @@ Change Logs
 0.3.0
 +++++
 
+* :pr:`28`: add syntax to check the readme syntax
 * :pr:`27`: fix missing __text_signature__ in docassert
 
 0.2.0
diff --git a/_doc/api/tools.rst b/_doc/api/tools.rst
index e6d2da8..ef56e9b 100644
--- a/_doc/api/tools.rst
+++ b/_doc/api/tools.rst
@@ -2,11 +2,28 @@
 tools
 =====
 
+
+Checks the readme syntax
+========================
+
+The command line checks the readme syntax in virtual environments.
+
+::
+
+    python -m sphinx_runpython readme -p <path_to_readme.rst> -v
+
+It is based on function:
+
+.. autofunction:: sphinx_runpython.readme.check_readme_syntax
+
+However, it is better to run command line ``twine check dist/*``
+assuming the whl was built by a command such as ``python setup.py sdist``.
+
 Convert notebooks into examples
 ===============================
 
 The command line converts every notebook in a folder
-into exmaples which can be used into a sphinx gallery.
+into examples which can be used into a sphinx gallery.
 
 ::
 
diff --git a/_unittests/ut__main/test_readme.py b/_unittests/ut__main/test_readme.py
index 8c17961..59e602b 100644
--- a/_unittests/ut__main/test_readme.py
+++ b/_unittests/ut__main/test_readme.py
@@ -1,5 +1,6 @@
 import os
 import unittest
+import sys
 from io import StringIO
 from contextlib import redirect_stdout
 from tempfile import TemporaryDirectory
@@ -8,6 +9,9 @@
 
 
 class TestReadme(ExtTestCase):
+    @unittest.skipIf(
+        sys.plarform == "win32", reason="Fails on windows due to tempoerary path"
+    )
     def test_venv_docutils08_readme(self):
         fold = os.path.dirname(os.path.abspath(__file__))
         readme = os.path.join(fold, "..", "..", "README.rst")

From 5a4346ad9e353717ee617b949214d62f39c7fe63 Mon Sep 17 00:00:00 2001
From: Xavier Dupre <xadupre@microsoft.com>
Date: Mon, 25 Sep 2023 12:37:13 +0200
Subject: [PATCH 3/3] misspelling

---
 _unittests/ut__main/test_readme.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_unittests/ut__main/test_readme.py b/_unittests/ut__main/test_readme.py
index 59e602b..c9e017b 100644
--- a/_unittests/ut__main/test_readme.py
+++ b/_unittests/ut__main/test_readme.py
@@ -10,7 +10,7 @@
 
 class TestReadme(ExtTestCase):
     @unittest.skipIf(
-        sys.plarform == "win32", reason="Fails on windows due to tempoerary path"
+        sys.platform == "win32", reason="Fails on windows due to tempoerary path"
     )
     def test_venv_docutils08_readme(self):
         fold = os.path.dirname(os.path.abspath(__file__))